{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 分类训练"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 机器学习的HelloWord，MNIST数据集，常用联系数据集 http://yann.lecun.com/exdb/mnist/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\x02'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# strcuct 二进制数据打包\n",
    "import struct\n",
    "\n",
    "struct.pack('>B',2)  # 打包类型 '>i' int类型 4个字节   B 1个字节 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b'\\x00\\x00\\x00\\x02'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "struct.pack('>i',2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2051, 60000, 28, 28)\n",
      "47040000\n"
     ]
    }
   ],
   "source": [
    "import struct\n",
    "with open('./MNIST/train-images-idx3-ubyte','rb')as f:\n",
    "    buffer = f.read(4*4) # 4个int  4*4 16个字节\n",
    "    head = struct.unpack('>iiii',buffer)\n",
    "    print(head)\n",
    "    length = head[1] * head[2] *head[3]\n",
    "    print(length)\n",
    "    buffer = f.read(length)\n",
    "    data = struct.unpack('>{}B'.format(length),buffer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "47040000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tuple"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "imgs = np.reshape(data, (head[1], head[2], head[3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imgs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt    #输出图片\n",
    "for i in range(5):\n",
    "    plt.imshow(imgs[i], cmap = 'gray')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  fetch_openml 方法导入mat文件数据\n",
    "sklearn版本大于等于2.0 使用 openml  低于使用mldata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "mnist = fetch_openml('mnist_784', version=1, cache=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data': array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.],\n",
       "        [0., 0., 0., ..., 0., 0., 0.]]),\n",
       " 'target': array(['5', '0', '4', ..., '4', '5', '6'], dtype=object),\n",
       " 'frame': None,\n",
       " 'feature_names': ['pixel1',\n",
       "  'pixel2',\n",
       "  'pixel3',\n",
       "  'pixel4',\n",
       "  'pixel5',\n",
       "  'pixel6',\n",
       "  'pixel7',\n",
       "  'pixel8',\n",
       "  'pixel9',\n",
       "  'pixel10',\n",
       "  'pixel11',\n",
       "  'pixel12',\n",
       "  'pixel13',\n",
       "  'pixel14',\n",
       "  'pixel15',\n",
       "  'pixel16',\n",
       "  'pixel17',\n",
       "  'pixel18',\n",
       "  'pixel19',\n",
       "  'pixel20',\n",
       "  'pixel21',\n",
       "  'pixel22',\n",
       "  'pixel23',\n",
       "  'pixel24',\n",
       "  'pixel25',\n",
       "  'pixel26',\n",
       "  'pixel27',\n",
       "  'pixel28',\n",
       "  'pixel29',\n",
       "  'pixel30',\n",
       "  'pixel31',\n",
       "  'pixel32',\n",
       "  'pixel33',\n",
       "  'pixel34',\n",
       "  'pixel35',\n",
       "  'pixel36',\n",
       "  'pixel37',\n",
       "  'pixel38',\n",
       "  'pixel39',\n",
       "  'pixel40',\n",
       "  'pixel41',\n",
       "  'pixel42',\n",
       "  'pixel43',\n",
       "  'pixel44',\n",
       "  'pixel45',\n",
       "  'pixel46',\n",
       "  'pixel47',\n",
       "  'pixel48',\n",
       "  'pixel49',\n",
       "  'pixel50',\n",
       "  'pixel51',\n",
       "  'pixel52',\n",
       "  'pixel53',\n",
       "  'pixel54',\n",
       "  'pixel55',\n",
       "  'pixel56',\n",
       "  'pixel57',\n",
       "  'pixel58',\n",
       "  'pixel59',\n",
       "  'pixel60',\n",
       "  'pixel61',\n",
       "  'pixel62',\n",
       "  'pixel63',\n",
       "  'pixel64',\n",
       "  'pixel65',\n",
       "  'pixel66',\n",
       "  'pixel67',\n",
       "  'pixel68',\n",
       "  'pixel69',\n",
       "  'pixel70',\n",
       "  'pixel71',\n",
       "  'pixel72',\n",
       "  'pixel73',\n",
       "  'pixel74',\n",
       "  'pixel75',\n",
       "  'pixel76',\n",
       "  'pixel77',\n",
       "  'pixel78',\n",
       "  'pixel79',\n",
       "  'pixel80',\n",
       "  'pixel81',\n",
       "  'pixel82',\n",
       "  'pixel83',\n",
       "  'pixel84',\n",
       "  'pixel85',\n",
       "  'pixel86',\n",
       "  'pixel87',\n",
       "  'pixel88',\n",
       "  'pixel89',\n",
       "  'pixel90',\n",
       "  'pixel91',\n",
       "  'pixel92',\n",
       "  'pixel93',\n",
       "  'pixel94',\n",
       "  'pixel95',\n",
       "  'pixel96',\n",
       "  'pixel97',\n",
       "  'pixel98',\n",
       "  'pixel99',\n",
       "  'pixel100',\n",
       "  'pixel101',\n",
       "  'pixel102',\n",
       "  'pixel103',\n",
       "  'pixel104',\n",
       "  'pixel105',\n",
       "  'pixel106',\n",
       "  'pixel107',\n",
       "  'pixel108',\n",
       "  'pixel109',\n",
       "  'pixel110',\n",
       "  'pixel111',\n",
       "  'pixel112',\n",
       "  'pixel113',\n",
       "  'pixel114',\n",
       "  'pixel115',\n",
       "  'pixel116',\n",
       "  'pixel117',\n",
       "  'pixel118',\n",
       "  'pixel119',\n",
       "  'pixel120',\n",
       "  'pixel121',\n",
       "  'pixel122',\n",
       "  'pixel123',\n",
       "  'pixel124',\n",
       "  'pixel125',\n",
       "  'pixel126',\n",
       "  'pixel127',\n",
       "  'pixel128',\n",
       "  'pixel129',\n",
       "  'pixel130',\n",
       "  'pixel131',\n",
       "  'pixel132',\n",
       "  'pixel133',\n",
       "  'pixel134',\n",
       "  'pixel135',\n",
       "  'pixel136',\n",
       "  'pixel137',\n",
       "  'pixel138',\n",
       "  'pixel139',\n",
       "  'pixel140',\n",
       "  'pixel141',\n",
       "  'pixel142',\n",
       "  'pixel143',\n",
       "  'pixel144',\n",
       "  'pixel145',\n",
       "  'pixel146',\n",
       "  'pixel147',\n",
       "  'pixel148',\n",
       "  'pixel149',\n",
       "  'pixel150',\n",
       "  'pixel151',\n",
       "  'pixel152',\n",
       "  'pixel153',\n",
       "  'pixel154',\n",
       "  'pixel155',\n",
       "  'pixel156',\n",
       "  'pixel157',\n",
       "  'pixel158',\n",
       "  'pixel159',\n",
       "  'pixel160',\n",
       "  'pixel161',\n",
       "  'pixel162',\n",
       "  'pixel163',\n",
       "  'pixel164',\n",
       "  'pixel165',\n",
       "  'pixel166',\n",
       "  'pixel167',\n",
       "  'pixel168',\n",
       "  'pixel169',\n",
       "  'pixel170',\n",
       "  'pixel171',\n",
       "  'pixel172',\n",
       "  'pixel173',\n",
       "  'pixel174',\n",
       "  'pixel175',\n",
       "  'pixel176',\n",
       "  'pixel177',\n",
       "  'pixel178',\n",
       "  'pixel179',\n",
       "  'pixel180',\n",
       "  'pixel181',\n",
       "  'pixel182',\n",
       "  'pixel183',\n",
       "  'pixel184',\n",
       "  'pixel185',\n",
       "  'pixel186',\n",
       "  'pixel187',\n",
       "  'pixel188',\n",
       "  'pixel189',\n",
       "  'pixel190',\n",
       "  'pixel191',\n",
       "  'pixel192',\n",
       "  'pixel193',\n",
       "  'pixel194',\n",
       "  'pixel195',\n",
       "  'pixel196',\n",
       "  'pixel197',\n",
       "  'pixel198',\n",
       "  'pixel199',\n",
       "  'pixel200',\n",
       "  'pixel201',\n",
       "  'pixel202',\n",
       "  'pixel203',\n",
       "  'pixel204',\n",
       "  'pixel205',\n",
       "  'pixel206',\n",
       "  'pixel207',\n",
       "  'pixel208',\n",
       "  'pixel209',\n",
       "  'pixel210',\n",
       "  'pixel211',\n",
       "  'pixel212',\n",
       "  'pixel213',\n",
       "  'pixel214',\n",
       "  'pixel215',\n",
       "  'pixel216',\n",
       "  'pixel217',\n",
       "  'pixel218',\n",
       "  'pixel219',\n",
       "  'pixel220',\n",
       "  'pixel221',\n",
       "  'pixel222',\n",
       "  'pixel223',\n",
       "  'pixel224',\n",
       "  'pixel225',\n",
       "  'pixel226',\n",
       "  'pixel227',\n",
       "  'pixel228',\n",
       "  'pixel229',\n",
       "  'pixel230',\n",
       "  'pixel231',\n",
       "  'pixel232',\n",
       "  'pixel233',\n",
       "  'pixel234',\n",
       "  'pixel235',\n",
       "  'pixel236',\n",
       "  'pixel237',\n",
       "  'pixel238',\n",
       "  'pixel239',\n",
       "  'pixel240',\n",
       "  'pixel241',\n",
       "  'pixel242',\n",
       "  'pixel243',\n",
       "  'pixel244',\n",
       "  'pixel245',\n",
       "  'pixel246',\n",
       "  'pixel247',\n",
       "  'pixel248',\n",
       "  'pixel249',\n",
       "  'pixel250',\n",
       "  'pixel251',\n",
       "  'pixel252',\n",
       "  'pixel253',\n",
       "  'pixel254',\n",
       "  'pixel255',\n",
       "  'pixel256',\n",
       "  'pixel257',\n",
       "  'pixel258',\n",
       "  'pixel259',\n",
       "  'pixel260',\n",
       "  'pixel261',\n",
       "  'pixel262',\n",
       "  'pixel263',\n",
       "  'pixel264',\n",
       "  'pixel265',\n",
       "  'pixel266',\n",
       "  'pixel267',\n",
       "  'pixel268',\n",
       "  'pixel269',\n",
       "  'pixel270',\n",
       "  'pixel271',\n",
       "  'pixel272',\n",
       "  'pixel273',\n",
       "  'pixel274',\n",
       "  'pixel275',\n",
       "  'pixel276',\n",
       "  'pixel277',\n",
       "  'pixel278',\n",
       "  'pixel279',\n",
       "  'pixel280',\n",
       "  'pixel281',\n",
       "  'pixel282',\n",
       "  'pixel283',\n",
       "  'pixel284',\n",
       "  'pixel285',\n",
       "  'pixel286',\n",
       "  'pixel287',\n",
       "  'pixel288',\n",
       "  'pixel289',\n",
       "  'pixel290',\n",
       "  'pixel291',\n",
       "  'pixel292',\n",
       "  'pixel293',\n",
       "  'pixel294',\n",
       "  'pixel295',\n",
       "  'pixel296',\n",
       "  'pixel297',\n",
       "  'pixel298',\n",
       "  'pixel299',\n",
       "  'pixel300',\n",
       "  'pixel301',\n",
       "  'pixel302',\n",
       "  'pixel303',\n",
       "  'pixel304',\n",
       "  'pixel305',\n",
       "  'pixel306',\n",
       "  'pixel307',\n",
       "  'pixel308',\n",
       "  'pixel309',\n",
       "  'pixel310',\n",
       "  'pixel311',\n",
       "  'pixel312',\n",
       "  'pixel313',\n",
       "  'pixel314',\n",
       "  'pixel315',\n",
       "  'pixel316',\n",
       "  'pixel317',\n",
       "  'pixel318',\n",
       "  'pixel319',\n",
       "  'pixel320',\n",
       "  'pixel321',\n",
       "  'pixel322',\n",
       "  'pixel323',\n",
       "  'pixel324',\n",
       "  'pixel325',\n",
       "  'pixel326',\n",
       "  'pixel327',\n",
       "  'pixel328',\n",
       "  'pixel329',\n",
       "  'pixel330',\n",
       "  'pixel331',\n",
       "  'pixel332',\n",
       "  'pixel333',\n",
       "  'pixel334',\n",
       "  'pixel335',\n",
       "  'pixel336',\n",
       "  'pixel337',\n",
       "  'pixel338',\n",
       "  'pixel339',\n",
       "  'pixel340',\n",
       "  'pixel341',\n",
       "  'pixel342',\n",
       "  'pixel343',\n",
       "  'pixel344',\n",
       "  'pixel345',\n",
       "  'pixel346',\n",
       "  'pixel347',\n",
       "  'pixel348',\n",
       "  'pixel349',\n",
       "  'pixel350',\n",
       "  'pixel351',\n",
       "  'pixel352',\n",
       "  'pixel353',\n",
       "  'pixel354',\n",
       "  'pixel355',\n",
       "  'pixel356',\n",
       "  'pixel357',\n",
       "  'pixel358',\n",
       "  'pixel359',\n",
       "  'pixel360',\n",
       "  'pixel361',\n",
       "  'pixel362',\n",
       "  'pixel363',\n",
       "  'pixel364',\n",
       "  'pixel365',\n",
       "  'pixel366',\n",
       "  'pixel367',\n",
       "  'pixel368',\n",
       "  'pixel369',\n",
       "  'pixel370',\n",
       "  'pixel371',\n",
       "  'pixel372',\n",
       "  'pixel373',\n",
       "  'pixel374',\n",
       "  'pixel375',\n",
       "  'pixel376',\n",
       "  'pixel377',\n",
       "  'pixel378',\n",
       "  'pixel379',\n",
       "  'pixel380',\n",
       "  'pixel381',\n",
       "  'pixel382',\n",
       "  'pixel383',\n",
       "  'pixel384',\n",
       "  'pixel385',\n",
       "  'pixel386',\n",
       "  'pixel387',\n",
       "  'pixel388',\n",
       "  'pixel389',\n",
       "  'pixel390',\n",
       "  'pixel391',\n",
       "  'pixel392',\n",
       "  'pixel393',\n",
       "  'pixel394',\n",
       "  'pixel395',\n",
       "  'pixel396',\n",
       "  'pixel397',\n",
       "  'pixel398',\n",
       "  'pixel399',\n",
       "  'pixel400',\n",
       "  'pixel401',\n",
       "  'pixel402',\n",
       "  'pixel403',\n",
       "  'pixel404',\n",
       "  'pixel405',\n",
       "  'pixel406',\n",
       "  'pixel407',\n",
       "  'pixel408',\n",
       "  'pixel409',\n",
       "  'pixel410',\n",
       "  'pixel411',\n",
       "  'pixel412',\n",
       "  'pixel413',\n",
       "  'pixel414',\n",
       "  'pixel415',\n",
       "  'pixel416',\n",
       "  'pixel417',\n",
       "  'pixel418',\n",
       "  'pixel419',\n",
       "  'pixel420',\n",
       "  'pixel421',\n",
       "  'pixel422',\n",
       "  'pixel423',\n",
       "  'pixel424',\n",
       "  'pixel425',\n",
       "  'pixel426',\n",
       "  'pixel427',\n",
       "  'pixel428',\n",
       "  'pixel429',\n",
       "  'pixel430',\n",
       "  'pixel431',\n",
       "  'pixel432',\n",
       "  'pixel433',\n",
       "  'pixel434',\n",
       "  'pixel435',\n",
       "  'pixel436',\n",
       "  'pixel437',\n",
       "  'pixel438',\n",
       "  'pixel439',\n",
       "  'pixel440',\n",
       "  'pixel441',\n",
       "  'pixel442',\n",
       "  'pixel443',\n",
       "  'pixel444',\n",
       "  'pixel445',\n",
       "  'pixel446',\n",
       "  'pixel447',\n",
       "  'pixel448',\n",
       "  'pixel449',\n",
       "  'pixel450',\n",
       "  'pixel451',\n",
       "  'pixel452',\n",
       "  'pixel453',\n",
       "  'pixel454',\n",
       "  'pixel455',\n",
       "  'pixel456',\n",
       "  'pixel457',\n",
       "  'pixel458',\n",
       "  'pixel459',\n",
       "  'pixel460',\n",
       "  'pixel461',\n",
       "  'pixel462',\n",
       "  'pixel463',\n",
       "  'pixel464',\n",
       "  'pixel465',\n",
       "  'pixel466',\n",
       "  'pixel467',\n",
       "  'pixel468',\n",
       "  'pixel469',\n",
       "  'pixel470',\n",
       "  'pixel471',\n",
       "  'pixel472',\n",
       "  'pixel473',\n",
       "  'pixel474',\n",
       "  'pixel475',\n",
       "  'pixel476',\n",
       "  'pixel477',\n",
       "  'pixel478',\n",
       "  'pixel479',\n",
       "  'pixel480',\n",
       "  'pixel481',\n",
       "  'pixel482',\n",
       "  'pixel483',\n",
       "  'pixel484',\n",
       "  'pixel485',\n",
       "  'pixel486',\n",
       "  'pixel487',\n",
       "  'pixel488',\n",
       "  'pixel489',\n",
       "  'pixel490',\n",
       "  'pixel491',\n",
       "  'pixel492',\n",
       "  'pixel493',\n",
       "  'pixel494',\n",
       "  'pixel495',\n",
       "  'pixel496',\n",
       "  'pixel497',\n",
       "  'pixel498',\n",
       "  'pixel499',\n",
       "  'pixel500',\n",
       "  'pixel501',\n",
       "  'pixel502',\n",
       "  'pixel503',\n",
       "  'pixel504',\n",
       "  'pixel505',\n",
       "  'pixel506',\n",
       "  'pixel507',\n",
       "  'pixel508',\n",
       "  'pixel509',\n",
       "  'pixel510',\n",
       "  'pixel511',\n",
       "  'pixel512',\n",
       "  'pixel513',\n",
       "  'pixel514',\n",
       "  'pixel515',\n",
       "  'pixel516',\n",
       "  'pixel517',\n",
       "  'pixel518',\n",
       "  'pixel519',\n",
       "  'pixel520',\n",
       "  'pixel521',\n",
       "  'pixel522',\n",
       "  'pixel523',\n",
       "  'pixel524',\n",
       "  'pixel525',\n",
       "  'pixel526',\n",
       "  'pixel527',\n",
       "  'pixel528',\n",
       "  'pixel529',\n",
       "  'pixel530',\n",
       "  'pixel531',\n",
       "  'pixel532',\n",
       "  'pixel533',\n",
       "  'pixel534',\n",
       "  'pixel535',\n",
       "  'pixel536',\n",
       "  'pixel537',\n",
       "  'pixel538',\n",
       "  'pixel539',\n",
       "  'pixel540',\n",
       "  'pixel541',\n",
       "  'pixel542',\n",
       "  'pixel543',\n",
       "  'pixel544',\n",
       "  'pixel545',\n",
       "  'pixel546',\n",
       "  'pixel547',\n",
       "  'pixel548',\n",
       "  'pixel549',\n",
       "  'pixel550',\n",
       "  'pixel551',\n",
       "  'pixel552',\n",
       "  'pixel553',\n",
       "  'pixel554',\n",
       "  'pixel555',\n",
       "  'pixel556',\n",
       "  'pixel557',\n",
       "  'pixel558',\n",
       "  'pixel559',\n",
       "  'pixel560',\n",
       "  'pixel561',\n",
       "  'pixel562',\n",
       "  'pixel563',\n",
       "  'pixel564',\n",
       "  'pixel565',\n",
       "  'pixel566',\n",
       "  'pixel567',\n",
       "  'pixel568',\n",
       "  'pixel569',\n",
       "  'pixel570',\n",
       "  'pixel571',\n",
       "  'pixel572',\n",
       "  'pixel573',\n",
       "  'pixel574',\n",
       "  'pixel575',\n",
       "  'pixel576',\n",
       "  'pixel577',\n",
       "  'pixel578',\n",
       "  'pixel579',\n",
       "  'pixel580',\n",
       "  'pixel581',\n",
       "  'pixel582',\n",
       "  'pixel583',\n",
       "  'pixel584',\n",
       "  'pixel585',\n",
       "  'pixel586',\n",
       "  'pixel587',\n",
       "  'pixel588',\n",
       "  'pixel589',\n",
       "  'pixel590',\n",
       "  'pixel591',\n",
       "  'pixel592',\n",
       "  'pixel593',\n",
       "  'pixel594',\n",
       "  'pixel595',\n",
       "  'pixel596',\n",
       "  'pixel597',\n",
       "  'pixel598',\n",
       "  'pixel599',\n",
       "  'pixel600',\n",
       "  'pixel601',\n",
       "  'pixel602',\n",
       "  'pixel603',\n",
       "  'pixel604',\n",
       "  'pixel605',\n",
       "  'pixel606',\n",
       "  'pixel607',\n",
       "  'pixel608',\n",
       "  'pixel609',\n",
       "  'pixel610',\n",
       "  'pixel611',\n",
       "  'pixel612',\n",
       "  'pixel613',\n",
       "  'pixel614',\n",
       "  'pixel615',\n",
       "  'pixel616',\n",
       "  'pixel617',\n",
       "  'pixel618',\n",
       "  'pixel619',\n",
       "  'pixel620',\n",
       "  'pixel621',\n",
       "  'pixel622',\n",
       "  'pixel623',\n",
       "  'pixel624',\n",
       "  'pixel625',\n",
       "  'pixel626',\n",
       "  'pixel627',\n",
       "  'pixel628',\n",
       "  'pixel629',\n",
       "  'pixel630',\n",
       "  'pixel631',\n",
       "  'pixel632',\n",
       "  'pixel633',\n",
       "  'pixel634',\n",
       "  'pixel635',\n",
       "  'pixel636',\n",
       "  'pixel637',\n",
       "  'pixel638',\n",
       "  'pixel639',\n",
       "  'pixel640',\n",
       "  'pixel641',\n",
       "  'pixel642',\n",
       "  'pixel643',\n",
       "  'pixel644',\n",
       "  'pixel645',\n",
       "  'pixel646',\n",
       "  'pixel647',\n",
       "  'pixel648',\n",
       "  'pixel649',\n",
       "  'pixel650',\n",
       "  'pixel651',\n",
       "  'pixel652',\n",
       "  'pixel653',\n",
       "  'pixel654',\n",
       "  'pixel655',\n",
       "  'pixel656',\n",
       "  'pixel657',\n",
       "  'pixel658',\n",
       "  'pixel659',\n",
       "  'pixel660',\n",
       "  'pixel661',\n",
       "  'pixel662',\n",
       "  'pixel663',\n",
       "  'pixel664',\n",
       "  'pixel665',\n",
       "  'pixel666',\n",
       "  'pixel667',\n",
       "  'pixel668',\n",
       "  'pixel669',\n",
       "  'pixel670',\n",
       "  'pixel671',\n",
       "  'pixel672',\n",
       "  'pixel673',\n",
       "  'pixel674',\n",
       "  'pixel675',\n",
       "  'pixel676',\n",
       "  'pixel677',\n",
       "  'pixel678',\n",
       "  'pixel679',\n",
       "  'pixel680',\n",
       "  'pixel681',\n",
       "  'pixel682',\n",
       "  'pixel683',\n",
       "  'pixel684',\n",
       "  'pixel685',\n",
       "  'pixel686',\n",
       "  'pixel687',\n",
       "  'pixel688',\n",
       "  'pixel689',\n",
       "  'pixel690',\n",
       "  'pixel691',\n",
       "  'pixel692',\n",
       "  'pixel693',\n",
       "  'pixel694',\n",
       "  'pixel695',\n",
       "  'pixel696',\n",
       "  'pixel697',\n",
       "  'pixel698',\n",
       "  'pixel699',\n",
       "  'pixel700',\n",
       "  'pixel701',\n",
       "  'pixel702',\n",
       "  'pixel703',\n",
       "  'pixel704',\n",
       "  'pixel705',\n",
       "  'pixel706',\n",
       "  'pixel707',\n",
       "  'pixel708',\n",
       "  'pixel709',\n",
       "  'pixel710',\n",
       "  'pixel711',\n",
       "  'pixel712',\n",
       "  'pixel713',\n",
       "  'pixel714',\n",
       "  'pixel715',\n",
       "  'pixel716',\n",
       "  'pixel717',\n",
       "  'pixel718',\n",
       "  'pixel719',\n",
       "  'pixel720',\n",
       "  'pixel721',\n",
       "  'pixel722',\n",
       "  'pixel723',\n",
       "  'pixel724',\n",
       "  'pixel725',\n",
       "  'pixel726',\n",
       "  'pixel727',\n",
       "  'pixel728',\n",
       "  'pixel729',\n",
       "  'pixel730',\n",
       "  'pixel731',\n",
       "  'pixel732',\n",
       "  'pixel733',\n",
       "  'pixel734',\n",
       "  'pixel735',\n",
       "  'pixel736',\n",
       "  'pixel737',\n",
       "  'pixel738',\n",
       "  'pixel739',\n",
       "  'pixel740',\n",
       "  'pixel741',\n",
       "  'pixel742',\n",
       "  'pixel743',\n",
       "  'pixel744',\n",
       "  'pixel745',\n",
       "  'pixel746',\n",
       "  'pixel747',\n",
       "  'pixel748',\n",
       "  'pixel749',\n",
       "  'pixel750',\n",
       "  'pixel751',\n",
       "  'pixel752',\n",
       "  'pixel753',\n",
       "  'pixel754',\n",
       "  'pixel755',\n",
       "  'pixel756',\n",
       "  'pixel757',\n",
       "  'pixel758',\n",
       "  'pixel759',\n",
       "  'pixel760',\n",
       "  'pixel761',\n",
       "  'pixel762',\n",
       "  'pixel763',\n",
       "  'pixel764',\n",
       "  'pixel765',\n",
       "  'pixel766',\n",
       "  'pixel767',\n",
       "  'pixel768',\n",
       "  'pixel769',\n",
       "  'pixel770',\n",
       "  'pixel771',\n",
       "  'pixel772',\n",
       "  'pixel773',\n",
       "  'pixel774',\n",
       "  'pixel775',\n",
       "  'pixel776',\n",
       "  'pixel777',\n",
       "  'pixel778',\n",
       "  'pixel779',\n",
       "  'pixel780',\n",
       "  'pixel781',\n",
       "  'pixel782',\n",
       "  'pixel783',\n",
       "  'pixel784'],\n",
       " 'target_names': ['class'],\n",
       " 'DESCR': \"**Author**: Yann LeCun, Corinna Cortes, Christopher J.C. Burges  \\n**Source**: [MNIST Website](http://yann.lecun.com/exdb/mnist/) - Date unknown  \\n**Please cite**:  \\n\\nThe MNIST database of handwritten digits with 784 features, raw data available at: http://yann.lecun.com/exdb/mnist/. It can be split in a training set of the first 60,000 examples, and a test set of 10,000 examples  \\n\\nIt is a subset of a larger set available from NIST. The digits have been size-normalized and centered in a fixed-size image. It is a good database for people who want to try learning techniques and pattern recognition methods on real-world data while spending minimal efforts on preprocessing and formatting. The original black and white (bilevel) images from NIST were size normalized to fit in a 20x20 pixel box while preserving their aspect ratio. The resulting images contain grey levels as a result of the anti-aliasing technique used by the normalization algorithm. the images were centered in a 28x28 image by computing the center of mass of the pixels, and translating the image so as to position this point at the center of the 28x28 field.  \\n\\nWith some classification methods (particularly template-based methods, such as SVM and K-nearest neighbors), the error rate improves when the digits are centered by bounding box rather than center of mass. If you do this kind of pre-processing, you should report it in your publications. The MNIST database was constructed from NIST's NIST originally designated SD-3 as their training set and SD-1 as their test set. However, SD-3 is much cleaner and easier to recognize than SD-1. The reason for this can be found on the fact that SD-3 was collected among Census Bureau employees, while SD-1 was collected among high-school students. Drawing sensible conclusions from learning experiments requires that the result be independent of the choice of training set and test among the complete set of samples. Therefore it was necessary to build a new database by mixing NIST's datasets.  \\n\\nThe MNIST training set is composed of 30,000 patterns from SD-3 and 30,000 patterns from SD-1. Our test set was composed of 5,000 patterns from SD-3 and 5,000 patterns from SD-1. The 60,000 pattern training set contained examples from approximately 250 writers. We made sure that the sets of writers of the training set and test set were disjoint. SD-1 contains 58,527 digit images written by 500 different writers. In contrast to SD-3, where blocks of data from each writer appeared in sequence, the data in SD-1 is scrambled. Writer identities for SD-1 is available and we used this information to unscramble the writers. We then split SD-1 in two: characters written by the first 250 writers went into our new training set. The remaining 250 writers were placed in our test set. Thus we had two sets with nearly 30,000 examples each. The new training set was completed with enough examples from SD-3, starting at pattern # 0, to make a full set of 60,000 training patterns. Similarly, the new test set was completed with SD-3 examples starting at pattern # 35,000 to make a full set with 60,000 test patterns. Only a subset of 10,000 test images (5,000 from SD-1 and 5,000 from SD-3) is available on this site. The full 60,000 sample training set is available.\\n\\nDownloaded from openml.org.\",\n",
       " 'details': {'id': '554',\n",
       "  'name': 'mnist_784',\n",
       "  'version': '1',\n",
       "  'format': 'ARFF',\n",
       "  'upload_date': '2014-09-29T03:28:38',\n",
       "  'licence': 'Public',\n",
       "  'url': 'https://www.openml.org/data/v1/download/52667/mnist_784.arff',\n",
       "  'file_id': '52667',\n",
       "  'default_target_attribute': 'class',\n",
       "  'tag': ['AzurePilot',\n",
       "   'OpenML-CC18',\n",
       "   'OpenML100',\n",
       "   'study_1',\n",
       "   'study_123',\n",
       "   'study_41',\n",
       "   'study_99',\n",
       "   'vision'],\n",
       "  'visibility': 'public',\n",
       "  'status': 'active',\n",
       "  'processing_date': '2018-10-03 21:23:30',\n",
       "  'md5_checksum': '0298d579eb1b86163de7723944c7e495'},\n",
       " 'categories': {},\n",
       " 'url': 'https://www.openml.org/d/554'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   1. DESCR数据集描述\n",
    "   2. data包含一个数组，每个实例为一行，每个特征为一列\n",
    "   3. target包含一个带有标签的数组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mnist['data'], mnist['target']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000, 784)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(70000,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_digit = X[16000]\n",
    "some_digit_image = some_digit.reshape(28,28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAOdUlEQVR4nO3dXYxUdZrH8d8jAhfOmCC0LAFcsCFmfYWxJJtolM24+HIhjsmskDhhIxnmQpMZmYtV1wjxRt0IE0xwtEcJzDqrgTACCWYdIKOEmwmFQUUJA2t6GV5CNxIzQsTx5dmLPm5a6Pqfps6pOgXP95N0quo8deo8HPrXp6r+depv7i4AF76Lqm4AQHsQdiAIwg4EQdiBIAg7EMTF7dzYuHHjfMqUKe3cJBBKb2+vjh8/bkPVCoXdzO6UtELSCEkvu/szqftPmTJF9Xq9yCYBJNRqtYa1pp/Gm9kISSsl3SXpaknzzezqZh8PQGsVec0+S9IBd//Y3f8m6XVJc8tpC0DZioR9oqS/DLp9KFv2HWa2yMzqZlbv7+8vsDkARRQJ+1BvApz12Vt373H3mrvXurq6CmwOQBFFwn5I0uRBtydJOlKsHQCtUiTsOyVNN7OpZjZK0jxJm8ppC0DZmh56c/evzOxhSW9pYOhtlbt/WFpnAEpVaJzd3d+U9GZJvQBoIT4uCwRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCFZnFFZ+jt7W1YmzRpUnLdiy9O/wps2LAhWd+6dWuyvnHjxoa1OXPmJNedO3dusn7llVcm69dee22yHk2hsJtZr6TPJH0t6St3r5XRFIDylXFk/yd3P17C4wBoIV6zA0EUDbtL+oOZ7TKzRUPdwcwWmVndzOr9/f0FNwegWUXDfrO7/0DSXZIeMrNbz7yDu/e4e83da11dXQU3B6BZhcLu7keyyz5Jb0iaVUZTAMrXdNjN7BIz+/631yXNkbSnrMYAlKvIu/HjJb1hZt8+zn+5+3+X0lUwzz33XLK+efPmZP29995rWNu3b19y3ZUrVybry5cvT9ZPnjyZrGe/H0NavXp1ct28ep7UOP1LL72UXPdCfMnZdNjd/WNJN5TYC4AWYugNCIKwA0EQdiAIwg4EQdiBIDjFtQ1Sp6BK+UNvfX19yfr06dMb1t5+++3kuuvXr0/WT506lax3stTpuXn7Zf/+/cn62LFjm2mpUhzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIxtnbYMeOHcl60a/rOnDgQMPavHnzkutec801yfptt92WrN9661lfTvQdRb7O+dNPP03WH3vssWT9k08+aVgbPXp0ct0RI0Yk6+cjjuxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7G0we/bsZL27uztZT42j53nnnXeS9euvvz5Zv/TSS5vedp7PP/88Wc+bsvnEiRNNb/v06dPJurs3/didiiM7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgTBOHsbTJo0KVm//fbbk/W87zBPueqqq5L1Vo6jS9LWrVsb1u65557kunlj4XlS56w/+OCDyXXHjBlTaNudKPfIbmarzKzPzPYMWnaZmW0xs/3Z5YW3Z4ALzHCexq+WdOcZyx6VtM3dp0valt0G0MFyw+7u2yWd+bnEuZLWZNfXSLq35L4AlKzZN+jGu/tRScouL290RzNbZGZ1M6sX/a41AM1r+bvx7t7j7jV3r3V1dbV6cwAaaDbsx8xsgiRll+lpRgFUrtmwb5K0ILu+QNLGctoB0Cq54+xm9pqk2ZLGmdkhSUskPSNprZktlHRQ0o9b2eSFLu+71c2s6frTTz+dXHfx4sXJet5Y9/PPP5+sv/rqqw1rX3zxRXLdvH93nmnTpjWsLVu2rNBjn49yw+7u8xuUflhyLwBaiI/LAkEQdiAIwg4EQdiBIAg7EASnuHaAm266KVnPO90yNbXxihUrkuseOXIkWT916lSyvnnz5mS9yPDZqFGjkvVbbrklWV+7dm3T274QcWQHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAYZ+8AeePsedMu33DDDU1ve926dU2vW1TeOPrKlSuT9byvg8Z3cWQHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAYZz8PjB07NlmfOHFiw9rhw4fLbuecpL7OOe988yKfH8DZOLIDQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCMs58Hdu7cmawfOnSoTZ2czd2T9ZkzZzasMY7eXrlHdjNbZWZ9ZrZn0LKlZnbYzHZnP3e3tk0ARQ3nafxqSXcOsfxX7j4j+3mz3LYAlC037O6+XdKJNvQCoIWKvEH3sJm9nz3NbzgZmZktMrO6mdX7+/sLbA5AEc2G/deSuiXNkHRU0rJGd3T3HnevuXutq6uryc0BKKqpsLv7MXf/2t2/kfQbSbPKbQtA2ZoKu5lNGHTzR5L2NLovgM6QO85uZq9Jmi1pnJkdkrRE0mwzmyHJJfVK+lkLe7zgbdiwIVm/7777kvUic6C3Wuo773ft2pVc98Ybbyy7ndByw+7u84dY/EoLegHQQnxcFgiCsANBEHYgCMIOBEHYgSA4xbUN9u7dm6w/8sgjLdv2RRel/57Pnj07Wd++fXuy/uWXXybrfX19DWvPPvtsct28r5rGueHIDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM7eBkuXLk3WDx482LJtX3fddcn6li1bkvU77rij0PopixcvbnpdnDuO7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBOPsF4Bp06Y1rG3cuLHQY3d3dyfrRcbZ8x4b5eLIDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBMM5egn379iXr69atK/T47p6sP/DAAw1rkydPTq771FNPJesvvvhisp7XW8oLL7yQrC9ZsqTpx8bZco/sZjbZzP5oZnvN7EMz+3m2/DIz22Jm+7PLMa1vF0CzhvM0/itJv3T3f5D0j5IeMrOrJT0qaZu7T5e0LbsNoEPlht3dj7r7u9n1zyTtlTRR0lxJa7K7rZF0b6uaBFDcOb1BZ2ZTJM2U9CdJ4939qDTwB0HS5Q3WWWRmdTOr9/f3F+sWQNOGHXYz+56k9ZJ+4e5/He567t7j7jV3r3V1dTXTI4ASDCvsZjZSA0H/nbv/Plt8zMwmZPUJkhpP1wmgcrlDb2Zmkl6RtNfdlw8qbZK0QNIz2WWxcynPY6+//nqyPrALW+ett95qWDt58mRy3TVr1iTrRXtPTQl9//33F3psnJvhjLPfLOknkj4ws93Zssc1EPK1ZrZQ0kFJP25NiwDKkBt2d98hqdGf9x+W2w6AVuHjskAQhB0IgrADQRB2IAjCDgRhRU5RPFe1Ws3r9XrbttcueVMuT506tdDj5/0ftXocPyV1eq0kvfzyyw1rI0eOLLud8Gq1mur1+pC/EBzZgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIvkq6BOPHj0/WFy5cmKznnQ+fd056K02cODFZf/LJJ5N1xtI7B0d2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCcfYSjB49Olnv6elJ1q+44opk/aOPPkrWT58+3bA2bdq05LoTJkxI1vPG2bu7u5N1dA6O7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQxHDmZ58s6beS/k7SN5J63H2FmS2V9FNJ/dldH3f3N1vV6IXsiSeeqLoFBDCcD9V8JemX7v6umX1f0i4z25LVfuXuz7WuPQBlGc787EclHc2uf2ZmeyWlP1YFoOOc02t2M5siaaakP2WLHjaz981slZmNabDOIjOrm1m9v79/qLsAaINhh93MvidpvaRfuPtfJf1aUrekGRo48i8baj1373H3mrvXurq6SmgZQDOGFXYzG6mBoP/O3X8vSe5+zN2/dvdvJP1G0qzWtQmgqNyw28AUoa9I2uvuywctH3y61I8k7Sm/PQBlGc678TdL+omkD8xsd7bscUnzzWyGJJfUK+lnLekQQCmG8278DklDzffMmDpwHuETdEAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSDM3du3MbN+Sf87aNE4Scfb1sC56dTeOrUvid6aVWZvf+/uQ37/W1vDftbGzeruXqusgYRO7a1T+5LorVnt6o2n8UAQhB0Iouqw91S8/ZRO7a1T+5LorVlt6a3S1+wA2qfqIzuANiHsQBCVhN3M7jSzfWZ2wMweraKHRsys18w+MLPdZlavuJdVZtZnZnsGLbvMzLaY2f7scsg59irqbamZHc723W4zu7ui3iab2R/NbK+ZfWhmP8+WV7rvEn21Zb+1/TW7mY2Q9GdJ/yzpkKSdkua7+0dtbaQBM+uVVHP3yj+AYWa3Sjop6bfufm227D8knXD3Z7I/lGPc/d86pLelkk5WPY13NlvRhMHTjEu6V9K/qsJ9l+jrX9SG/VbFkX2WpAPu/rG7/03S65LmVtBHx3P37ZJOnLF4rqQ12fU1GvhlabsGvXUEdz/q7u9m1z+T9O0045Xuu0RfbVFF2CdK+sug24fUWfO9u6Q/mNkuM1tUdTNDGO/uR6WBXx5Jl1fcz5lyp/FupzOmGe+YfdfM9OdFVRH2oaaS6qTxv5vd/QeS7pL0UPZ0FcMzrGm822WIacY7QrPTnxdVRdgPSZo86PYkSUcq6GNI7n4ku+yT9IY6byrqY9/OoJtd9lXcz//rpGm8h5pmXB2w76qc/ryKsO+UNN3MpprZKEnzJG2qoI+zmNkl2RsnMrNLJM1R501FvUnSguz6AkkbK+zlOzplGu9G04yr4n1X+fTn7t72H0l3a+Ad+f+R9O9V9NCgryslvZf9fFh1b5Je08DTui818IxooaSxkrZJ2p9dXtZBvf2npA8kva+BYE2oqLdbNPDS8H1Ju7Ofu6ved4m+2rLf+LgsEASfoAOCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIP4PVnVUohDClDUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(some_digit_image, cmap =matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'8'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[16000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "### 建立测试集和训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([29991,   921,  4441, ..., 48009, 15426, 37862])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将数据集合交叉洗牌，交叉验证时，每个子集数据分布均匀，有些机器学习算法对顺序敏感 = \n",
    "import numpy as np\n",
    "shuffle_index = np.random.permutation(60000)\n",
    "shuffle_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = X_train[shuffle_index], y_train[shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['3', '5', '1', ..., '2', '4', '1'], dtype=object)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 训练二元分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 识别数字8，分类为8和非8\n",
    "# 创建目标向量\n",
    "y_train_8 = (y_train == '8')\n",
    "y_train_8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       ...,\n",
       "       [ True, False, False, ..., False, False, False],\n",
       "       [False,  True, False, ..., False,  True, False],\n",
       "       [False, False, False, ..., False, False, False]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_8.reshape(20,-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, ..., False, False, False])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_8 = (y_test == '8')\n",
    "y_test_8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False,  True, False],\n",
       "       [False, False, False, ..., False, False,  True],\n",
       "       ...,\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False],\n",
       "       [False, False, False, ..., False, False, False]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_8.reshape(20, -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SGD 梯度下降 线性分类器， 适合非常大的数据，可以独立处理数据，一次一个，适合在线学习\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_clf = SGDClassifier(random_state = 42)\n",
    "sgd_clf.fit(X_train, y_train_8)\n",
    "\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 性能考核"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用交叉验证测量精度\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9368, 0.9011, 0.9321])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 评估分类器比评估回归器要困难的多\n",
    "\n",
    "# cv 代表3个折叠，将训练集分成3份，测试3次，每次2份训练，一份测试\n",
    "# 正确率达到95%以上\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "cross_val_score(sgd_clf, X_train, y_train_8 , cv = 3, scoring = 'accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 把每张图都分类成 非8\n",
    "from sklearn.base import BaseEstimator\n",
    "class Never8Classifier(BaseEstimator):\n",
    "    def fit(self, X, y=None):\n",
    "        pass\n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype = bool)   #其实是写死的代码，没有分类效果。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[False],\n",
       "       [False],\n",
       "       [False],\n",
       "       ...,\n",
       "       [False],\n",
       "       [False],\n",
       "       [False]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.zeros((len(X), 1), dtype = bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.90055, 0.90315, 0.90375])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "never_8_clf = Never8Classifier()\n",
    "cross_val_score(never_8_clf, X_train, y_train_8, cv=3, scoring = 'accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   1. 写死的分类器，never_8_clf，交叉验证准确率也是90%以上，有问题，主要是8的数字占比10%左右，所以90%的时间都是正确的。\n",
    "   2. 说明准确率无法成为分类器的首要性能指标。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 混淆矩阵"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 评估分类器性能的最好方法是混淆矩阵\n",
    "# A类别实例被分为B类别次数 \n",
    "# 想要知道分类学将数值3和数值5混淆多少次，通过混淆矩阵的5行3列\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "y_train_pred = cross_val_predict(sgd_clf, X_train, y_train_8, cv =3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 与cross_val_scors相比\n",
    "   1. 同样执行交叉验证\n",
    "   2. 返回的不是评估分数，是每个折叠的预测\n",
    "   3. 每个实例在模型预算时使用的数据，在训练期间从没见过\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[51409,  2740],\n",
       "       [ 1860,  3991]], dtype=int64)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_train_8, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 行表示实际类别，列表示预测类别\n",
    "# 第一行 第一列 51640 被正确的分为非8  真负类\n",
    "# 第一行 第二列 2059  被错误的分成 8 假正类\n",
    "# 第二行 第一列 1966  被错误的分为 非8 假负类\n",
    "# 第二行第二列 3885  被正确的分在 8  真正类\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[54149,     0],\n",
       "       [    0,  5851]], dtype=int64)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_predict = y_train_8\n",
    "confusion_matrix(y_train_8, y_train_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 正类预测的准确率  被称为分类器的精度\n",
    "$\n",
    "精度=\\cfrac{TP}{TP+FP}\n",
    "$\n",
    "\n",
    "TP 是真正类，FP是假正类\n",
    "\n",
    "$\n",
    "召回率TPR=\\cfrac{TP}{TP+FN}\n",
    "$\n",
    "\n",
    "FN是假负类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 精度和召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5929282424602585"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score\n",
    "\n",
    "precision_score(y_train_8, y_train_pred)    # y_train_8 实际标签  y_train_pred 混淆矩阵 预测标签  精度 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6821056229704324"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_8, y_train_pred)  # 召回率 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 精度 说明 检测一张图片。68%是准确的， 召回 只有55%概率数字8被检测出来\n",
    "# 精度和召回合成单一指标， 成功F1分数，谐波平均值\n",
    "# 平均值平均对待所有的值，谐波平均值会给给予较低值更高的权重，只有召回率和精度都很高的时候，才能获得较高F1分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6343983468446989"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(y_train_8, y_train_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# F1对那些具有相近的精度和召回率 的分类器更有利，\n",
    "# 训练一个分类器检查儿童可以放心观看的视频，你可能要求拦截很多的好视频，低召回率，保留下来都是安全的视频，高精度\n",
    "# 不能同时增加精度并减少召回率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 精度/召回率权衡"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   1. SGDClassifier 对每个实例基于决策函数计算一个分值，大于阀值为正类，否则为负类\n",
    "   2. 提高阀值，精度提高，召回率降低\n",
    "   3. sklearn不能直接设置阀值，可以访问决策分数\n",
    "   4. SGDClassifier默认阀值是0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如何设置阀值\n",
    "# 用predict_proba得到买个实例属于正类的概率，然后对概率切一下，以LogisticRegression为例\n",
    "# clf= LogisticRegression()\n",
    "# clf.fit(X_train, y_train)\n",
    "# pred_proba = clf.predict_proba(X-test)[:1]\n",
    "# threshold = 0.75 阀值设置为0.75\n",
    "# pred_labe1 = pred_proba > threshold\n",
    "\n",
    "# pred_proba 是每个实例的为真的概率\n",
    "# 假设阀值是0.75\n",
    "# pred_lable1里true就是概率大于0.75的\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-174.12640048])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 返回决策值decision_function\n",
    "y_scores = sgd_clf.decision_function([some_digit])\n",
    "y_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 0\n",
    "y_some_digit_pred = (y_scores > threshold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 提高阀值可以降低召回率，就错过了这个图，即认为这个不是8\n",
    "threshold = 10000\n",
    "y_some_digit_pred = (y_scores > threshold)\n",
    "y_some_digit_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 如何决定使用什么阀值\n",
    "# cross_val_predict一般返回预测结果，现在设置method的值，使用返回决策值\n",
    "# 对训练集进行交叉验证得到决策值\n",
    "y_scores = cross_val_predict(sgd_clf, X_train, y_train_8, cv =3, method = 'decision_function')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000,)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 有了决策值y_scores，可以计算所有可能的阀值的精度和召回\n",
    "y_scores.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_8, y_scores)  # precisions 精度 recalls召回 thresholds 阀值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAEPCAYAAABx8azBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3xUZfr//9eVSSH0XkMTaYoCEroCKk1YcV392fWz7lfBVXBVlo+76+5i29W17+qyiCtiwY8dbICKDUUpAQWRIlVCDU1Aasr9++MMySRMSIBJTmbm/eQxjznnPvc557onE66cdt/mnENERET8k+B3ACIiIvFOyVhERMRnSsYiIiI+UzIWERHxmZKxiIiIz5SMRUREfFZiMjaziWaWZWZLilluZvYvM1tlZovN7KzIhykiIhK7SnNkPAkYfIzlFwCtg6/hwH9OPiwREZH4UWIyds7NAnYeo8pFwAvOMweoaWaNIhWgiIhIrEuMwDaaAJkh8xuCZZuLVjSz4XhHz1SpUqVLu3btIrB7iTUOx+Iti8nJywm7vFG1RjSu1hiA3Qd3s2rnqmK3dWbDM0lKSAJg1c5V7D6421tgkBxIJtESCSQEqFGpBg2qNAAgOy+brT9vJSmQREoghdTEVBIDiQQsEMFWSkkOHoTsbKhUCZKSYPly2LcvfN06daBFC2/6wAFYurT47bZtC1WretOZmZCV5U03aQING0YsfJGwFixYsN05V69oeSSSsYUpC9vHpnNuAjABID093WVkZERg9xJrdh/czavfv0puXi65Lpc8l1doumdaT85pfg4AS7KWMGHBBH4+/DPZedn59XLycjice5gXL36RmpVqAnDbjNv4cv2X7Dywk3U/reNw8B/Arzr+ikm/nATA4q2L6Ti+41FxJSUk0ahaI2b9ehbNazYHYMGmBVRKrES7uu0IJChZn6yvv4aLLoIuXeCjjyA3F8aPhxEjYMYM2L49/HqnnAK9ennTP/0E771X/D4GDYJ6wf8K58+HFSu86c6d4fTTI9cWkXDM7Mdw5ZFIxhuApiHzacCmCGxX4lSNSjUY3mV4qep2qN+Bf13wr1LVfWLwE/nTew7tYdu+bWzfv529h/dSq1Kt/GUNqzbk7+f9nfW717Ns+zJ+3P0j2/ZtY1/2PtbvXk9yIDm/7h8+/gMz18ykSlIVOjXsRO+mvemR1oMLWl9ApcRKpWyxHPHVV7Btm5d4jzh0yHsffKw7V0LUrAnXXFO6ul27ei8Rv0UiGb8DjDSzV4DuwG7n3FGnqEUqkuop1ameUp1WtVsdtax+lfr88Zw/HlW+P3s/K7avoG7luvll7eq0Y/n25WzYs4HZmbOZnTkbgNTEVJ4a8hS/6fybsmtEDOrd2ztdfPgwVK8Oq1d7R8kisa7EZGxm/wf0A+qa2QZgLJAE4JwbD0wDhgCrgP3A9WUVrIifKidVpnOjzoXKnhzyJE8OeZK1u9byVeZXfL/te95e8TZLty2leY3m+fXe++E9KiVWov8p/cs77KjgnHeNePlyuOQSuOMOaNYMdu70rgeLxDrzawhFXTOWWLZs2zJa12lNYoL39+6glwbx4eoPGXDKAEZ1G8WFbS/0OcKK44orYOZM2LHDmx8zBh56yN+YRMqKmS1wzqUXLVcPXCJloH299vmJ2DnHGfXPIDmQzEdrPmLYK8No+1RbnlnwDLl5uT5HWv727vVuyJo3z5t/442CRAwwdao/cYn4SclYpIyZGY8MfITM2zMZ02sMNVJq8MOOHxj+3nAue+My8lye3yGWm6VLvWvBEyZA9+5gBhdcULjOyy/7E5uInyJxA1eZ2b17N9u3b+fw4cN+hyLlLDk5mbp161KjRg2/Q4mY+lXq89CAh7in3z08+82z3DbjNlICKSRY7P5N7BwklNC8Dh3g3Xdh1izvTugzzyyf2EQqkgqbjA8ePMjWrVtJS0sjNTUVs3CPM0sscs5x4MABNmzYQEpKCpUqxdYjQqlJqYzsNpLeTXsXOirevHczqUmp+c9FR7O1a71nf8MJBOD++71ngWfPhlGjvPI+fcovPpGKpsLewJWZmUnVqlWpVatWsXUktu3cuZN9+/bRtGnTkitHuczdmZz7/LlUSa7Cx9d9XOjxqWizb19BD1fhzJnjnaIWiUdRdwPXwYMHqXqs32iJedWqVePgwYN+h1Eu9mXvY+/hvfm9f83bOM/vkE5YWlrh+TVrvNPVR15KxCJHq7DJOCcnh8TECnsWXcpBYmIiOTnh+6eONe3qtuOL67+gfd32bNq7ib6T+jJ58WS/wzoumzd73VfOnl1Q5hy0bOlfTCLRosImY0DXieNcvP3829RpQ8bwDIa1HcbBnINcM+Uapq2c5ndYpWIGjRtDYqLXv/P+/V4iFpHSqdDJWCTeVE6qzNTLp/KbTl43mle/dXXBSFMV1NixR5dtVoe4IsdF54FFKhgz45lhz1ArtRYjuoygRqWK/XhXkyaF5wcPLv5OahEJT0fG5WjSpEmYWf6rWrVqdOzYkaeeeqpcr43efffdx30KuF+/fvTr169sApKjJFgCjwx8hNZ1WvsdyjGZeb1pbdgAeXneqenp0/2OSiT66MjYB6+//jppaWns2bOH119/nVGjRpGVlcW9995bLvu/4YYbGFza8eiCxo0bV0bRSElW71zNfbPu46eDPzH1iorTV2To33Pr1h19hCwipadk7INOnTpx6qmnAjBw4EBWrVrFE088ETYZO+fIzs4mOTn5qGUnKi0tjbSiz5+U4LTTTovY/uX4BBICvLXsLfYe3sszC57hxi43+h0SX39deL53b3/iEIkVOk1dAXTt2pW9e/eSlZVFixYtuOaaa5g4cSLt2rUjOTmZ999/H4D9+/dz55130rJlS5KTk2nZsiV/+9vfyMsr3Lfxtm3buPnmm2natCkpKSk0bdqUa6+9lkPBUdrDnab+5z//Sfv27UlNTaVWrVqkp6czZcqU/OXhTlOvWLGCiy++mJo1a5KamkqPHj2YEToqfMi+Vq5cydChQ6latSrNmzfn3nvvPSpuCa9FzRb87by/AXDP5/ewY/+OEtYoe716FUyvWeNfHCKxIiqTsVnxrwkTCupNmHDsuqG6dCm+3vDhBfUWLIh8e9auXUsgEMjv5OTTTz/lscceY+zYscyYMYMzzzyTnJwcBg0axH//+19+97vfMX36dG644Qbuu+8+xowZk7+tXbt20atXL1599VXuuOMOpk2bxkMPPUR2dnaxfXxPnjyZ0aNHc+WVVzJt2jQmT57MpZdeys6dO4uNedOmTZx99tksWrSIp556itdee42aNWsydOhQpoe5aHjxxRdz3nnnMXXqVH75y18yduxYnn/++ZP85OLHzV1vpn3d9mzcu5F7Pr/H11hCH1n63e/0HLFIRDjnfHl16dLFHcvSpUuLXVa4P5/Cr6efLqj39NPHrhvqrLOKr3fjjQX1MjKOGfYxPffccw5wy5cvd9nZ2W7nzp1u/PjxLiEhwV100UXOOeeaN2/uUlNT3ebNmwut+8ILLzjAff7554XK77//fpeUlOS2bt3qnHPuL3/5i0tISHALFy4sNo6xY8c6Qj6AW265xXXu3PmYsfft29f17ds3f3706NEuEAi4lStX5pfl5OS4Nm3aFNrWkX1NnDix0PY6dOjgBgwYcMx9Onfs70G8+Wr9V467cQn3JLjFWxb7Fsfs2c41a+ZcjRrO5eX5FoZIVAIyXJicGJVHxsdKsaFHscOHH7tuqAULiq8XerTdpcvJx9+uXTuSkpKoXbs2N998M1dffTUTJ07MX96jRw8aNmxYaJ0ZM2bQvHlzevXqRU5OTv5r4MCBZGdnM2fOHAA+/PBDunbtSufOnUsdT9euXfn2228ZNWoUM2fOZP/+/SWuM2vWLHr06JF/7RsgEAhw5ZVX8u2337Jnz55C9YcOHVpovkOHDqxfv77UMQr0bNqT6ztdT57L45Zpt/gy9KKZd324Th346aejzzCJyInRDVw+mDJlCmlpaVSrVo3mzZsfNSpRo0aNjlonKyuLH3/8kaSkpLDb3BEcnX3Hjh107NjxuOK57rrrOHjwIM8++yzjxo0jKSmJIUOG8Nhjj9GiRYuw6+zcuTNswm/YsCHOOXbt2kX16tXzy2vXrl2oXkpKStz0Ox1JDw14iG+2fMMVHa4gNy+XhED5/T0deon/m2/KbbcicUHJ2AcdOnQodERZVLhngOvUqUPLli157bXXwq5zJGnWrVuXjRs3Hlc8ZsaIESMYMWIEu3bt4sMPP2T06NFcfvnlzJ07N+w6tWvXZsuWLUeVb9myBTM7KvlKZNStXJcFwxf4MgZyIFAw/eOP5b57kZgWlaep49HgwYPzh5VMT08/6lW3rjfk3sCBA5k3bx6LFi06of3UqlWLyy+/nMsuu4wlS5YUW69v377MmTOHdevW5Zfl5uby6quv0rlzZ6pVq3ZC+5eShSbiQzmHymWfoZd1KleGZs3KZbcicUNHxlHi6quv5rnnnuP8889n9OjRdOzYkcOHD7N69Wreeecdpk6dSuXKlbn99tt5+eWX6d+/P3/+858544wz2L59O2+//Tbjx48PmySHDx9OtWrV6NmzJ/Xr1+eHH37gxRdfZODAgcXGc/vttzNp0iQGDBjAPffcQ/Xq1Rk3bhw//PBD/qNYUnYOZB9g7Gdjefm7l1l00yLqVK5Tpvvbt69g+hg32YvICVIyjhJJSUl88MEHPPjgg0yYMIG1a9dSpUoVWrVqxdChQ/M7BalZsyazZ8/mz3/+Mw8++CA7duygQYMGnHfeecV2HNK7d2+ee+45XnzxRXbv3k3jxo255ppruOee4h+hady4MV9++SV33nknv/3tbzl06BCdOnXi/fffP+7eveT4JQWS+GD1B2zcu5G/f/F3Hh30aJnur2pV2LYNEhIgJaVMdyUSl8z5NM5Zenq6y8jIKHb5smXLaN++fTlGJBWRvgfF+yrzK3pP7E215Gps+f0WKidV9jskESmBmS1wzqUXLdc1Y5Eo1atpL7o27srew3t57fvwN/ZFwvbt8PbbhU9Vi0hkKRmLRLHrO10PwONzHqesznLdfTf88pdw9dVlsnkRQclYJKpd2/Fa6lWux+Kti/ly/Zdlso9//9t71x3UImVHyVgkilVNrsr1na6nWnI1kgLhO4Q5GaEdqY0aFfHNi0iQ7qYWiXI3nHUDQ9sMpUdaj4hvO7Qr2NatI755EQlSMhaJcq3rtKZ1nbLJlCEDgolIGdJpapEYsefQHj5Z+0mZDCBx880R36SIhFAyFokBuXm5tH2qLee/cD4zVs2I2HYXLoRKleDxxyO2SREJQ8lYJAYEEgL8ptNvAHh96esR2ea778JHH3nPFxfTeZuIRIiSsUiMuPpM70Hgt5a9RXZu9klvb9gwuPNOOM5BwETkBCgZl6NJkyZhZvmv5ORkWrVqxZ/+9Cffx/Zt0aIFv/71r/Pnj8QaOiqTVGyn1TuN1rVbs+fQHj5d9+lJbWv69ILpMMNri0iEKRn74PXXX+frr7/m/fffZ9CgQTzwwAOM0W2rEgEXt7sYgOkrp5dQ89juuKNgOlHPXIiUuVIlYzMbbGYrzGyVmf0hzPIaZvaumS0ys+/N7PrIhxo7OnXqRI8ePRgwYADjxo2jf//+PPvss+TlRf4uWIkvw9oOA2DK8ikn3D1mdjYsX+5Nq6MPkfJRYjI2swDwb+AC4DTgSjM7rUi1W4ClzrmOQD/gUTPTLR+ldNZZZ3HgwAG2b9+eX7Z27Vquvvpq6tWrR0pKCp06dWLKlClHrbto0SIuvvhi6tSpQ2pqKm3btuWBBx7IX/7hhx8yZMgQGjVqROXKlenQoQOPPvooubm55dI2KV890nrQpFoTWtdpze5Du09oG2+8UTA9cmSEAhORYyrNCahuwCrn3BoAM3sFuAhYGlLHAdXMzICqwE4gJ8Kx5rN7rNhlT//iaYZ3GQ7AhAUTGPHeiGLrurEFRw5dJnRh4eaFYevdeNaNTLjQ64powaYFdGnc5UTCLta6deuoUaMGdep4A8RnZmbSvXt36tevz+OPP069evV49dVXueSSS5g6dSrDhnlHP/PmzaNfv36ceuqpPP7446SlpbFy5UoWL16cv+01a9Zw/vnnM2rUKCpVqkRGRgZ3330327Zt48EHH4xoO8R/gYQAq29dTUriiQ86/NlnBdNt2px8TCJSstIk4yZAZsj8BqB7kTpPAe8Am4BqwOXOHd3zgJkNB4YDNIvjXudzc3PJyclh7969TJkyhTfffJMnnniCQCAAwN13341zjs8//zw/QQ8aNIjMzEz++te/5ifj3//+99SpU4c5c+ZQubI3lu15551XaF833XRT/rRzjnPOOYfDhw/zyCOP8Pe//52EBN02EGtOJhED3HUXZGXBbbdFKCARKVFpknG4w9CiF6MGAd8C5wGtgI/M7Avn3J5CKzk3AZgAkJ6efsLjvYUe0R7L8C7D84+SS7Jg+IJS1YvEUXG7du0Kzd98882MDDkfOGPGDIYMGUKNGjXIySk4wTBo0CDGjBnDnj17SExMZPbs2YwZMyY/EYezefNm7r77bmbMmMGmTZsKbS8rK4uGDRuedHuk4jmUc4iMTRm0q9uOOpXrHNe6zZpBmCsiIlKGSnNYtAFoGjKfhncEHOp64C3nWQWsBdohYU2ZMoX58+czbdo0+vfvz7hx43jhhRfyl2dlZfHCCy+QlJRU6HXkjusdO3awa9cu8vLySEtLK3Y/eXl5DBs2jPfee48///nPfPLJJ8yfP5+77roLwPfHqaTsXPbGZZz93NlM/m7yca23fTssXQplNDSyiBSjNEfG84HWZtYS2AhcAVxVpM564HzgCzNrALQF1kQy0FjSoUMHTj31VMA7rXzmmWcyZswYLrnkEqpUqUKdOnU455xzuPPOO8Ou37hxY3Jzc0lISGDjMXpkWL16NRkZGbz44otcc801+eXvvvtuZBskFc6v2v2Kd1a8w5vL3uTW7reWer3XXoNbboEbbyw8YpOIlK0Sj4ydcznASOADYBnwmnPuezO7ycyOXJC8D+hlZt8BHwN3Oue2h9+ihEpJSeHhhx8mKyuLcePGATB48GAWL17M6aefTnp6+lGvlJQUKleuzNlnn81LL73EgQMHwm57//79ACQlFYxzm52dzeTJx3e0JNFn8KmDAZizYQ4Hc0p/BmTmTO+9W7eyiEpEilOqx/mdc9OAaUXKxodMbwIGRja0+DFs2DC6du3KI488wsiRI7n33nvp1q0bffr0YeTIkbRo0YJdu3axZMkS1qxZw8SJEwF45JFH6Nu3Lz179mT06NGkpaWxZs0avv32W5588knat29P8+bNueuuuwgEAiQlJfG4evyPCw2qNqBD/Q4syVrC/I3zOaf5OSWuk51dcK24f/8yDlBECtGttBXE/fffT1ZWFuPHj6dZs2ZkZGTQsWNH/vSnPzFgwAB++9vf8vnnnxe6W7pr167Mnj2bpk2bMmrUKIYMGcLDDz+cfx05OTmZqVOn0rBhQ6677jpuueUW+vTpwx/+cFS/LRKD+jTrA8DHaz8uVf2XXy6YbtGiDAISkWLZifbSc7LS09NdRkZGscuXLVtG+/btyzEiqYj0PThx7/3wHhf+34V0atiJb0Z8U2L9Vq1gTfBOD93AJVI2zGyBcy69aLmOjEVi1Pktz6dSYiWAUl03Dt5iwKOPlmVUIhKOuoAXiVGpSalsvGMjtVNrl6p+/fqwZQtcd10ZByYiR1EyFolhpU3EAIsWQV4eqFM2kfKnXzuRGOecY/XO1WTnZpdYV4lYxB8V+lfPr5vLpGLQzz8yzn3+XE598lRmZ84uts66dXDoUPnFJCKFVdhknJSUVGxnFhIfDhw4UKjDEjkxp9c7HYCZa2YWW+cXv4Dq1SFkwC8RKUcVNhnXr1+fjRs3sn//fh0hxRnnHPv372fjxo3Ur1/f73Ci3i/a/AIo/nnjPXsK+qPWkIki/qiwN3BVr14dgE2bNpGdXfK1LoktSUlJNGjQIP97ICeuV9NeJCYkMn/jfHYd2EWt1FqFli9Y4CXijh2hUiWfghSJcxU2GYOXkPWfscjJqVGpBn2a9+GTtZ8wfdV0rjqj8Dgvc+d67+qPWsQ/FfY0tYhEzvktzwfg68yvj1p2JBl3716eEYlIKCVjkTjQp7nXT3XRO6qdUzIWqQgq9GlqEYmMnmk9mfP/5tClcZdC5Zs2webNUKMGtG7tU3AiomQsEg8CCQG6px196FujBrRt691FrQ4/RPyjZCwSZ/JcHgnmZd6qVeHrr71njEXEP/pbWCROrNm1hn6T+tFvUr/8sn37IDkZAgH/4hIRHRmLxI26lesy68dZJCYkciD7ACmBVKpW9Zbt2gU1a/obn0g805GxSJyonlKdMxqcQXZeNgs2L2D9+oJlNWr4F5eIKBmLxJWeaT0B73njlSu9snPOATMfgxIRJWOReNKraS8AvtrwFdOmeWU6Khbxn64Zi8SR0CPj3YsdYJxyir8xiYiOjEXiyqm1T6Vu5bps3beVzQfWAXDZZf7GJCI6MhaJK2bG2L5jSQ1U4ebHvdunO3TwOSgRUTIWiTcju40kLw+6zoKVK3XNWKQiUDIWiUMJCXDmmd5LRPyna8YiceiNpW9w6/Rb+fnwz36HIiLoyFgkLt36+oNsZgHpVS/munPO9TsckbinI2OROHTgB+8Rp7mbvvI5EhEBJWORuHPwIOxe2g2AH3Pm+RyNiICSsUjcWb4cXKaXjBdsmYdzzueIRETJWCTOLFkC7GxNSnZ9tvy8haXblvodkkjcUzIWiTNLlgAugdaJ5wPwxfov/A1IRHQ3tUi8WbLEe+/VuB+ptoqalTSQsYjfSpWMzWww8E8gAPzXOfdgmDr9gCeAJGC7c65vBOMUkQg56yzYvRtG9xtOmzbD/Q5HRChFMjazAPBvYACwAZhvZu8455aG1KkJjAMGO+fWm1n9sgpYRE7Ovff6HYGIFFWaa8bdgFXOuTXOucPAK8BFRepcBbzlnFsP4JzLimyYIlIWcvNy+WbzN6zeudrvUETiWmmScRMgM2R+Q7AsVBuglpl9ZmYLzOy6cBsys+FmlmFmGdu2bTuxiEXkhK1ZA0uXQna2N3//rPs5a8JZjJs/zt/AROJcaZKxhSkr+mBiItAFGAoMAv5iZm2OWsm5Cc65dOdcer169Y47WBE5Of/8J5x+Ojz2mDffq2kvAGatn+VjVCJSmmS8AWgaMp8GbApTZ4Zzbp9zbjswC+gYmRBFJFKO3El9xhnee8+mPQlYgG82f8Pug7v9C0wkzpUmGc8HWptZSzNLBq4A3ilS523gHDNLNLPKQHdgWWRDFZGT9d133nuHDt571eSqdGnchVyXy/xN8/0LTCTOlZiMnXM5wEjgA7wE+5pz7nszu8nMbgrWWQbMABYD8/Aef1pSdmGLyPHKyoJt26BaNWgacq6re5PuAMzbqH6qRfxSqueMnXPTgGlFysYXmX8YeDhyoYlIJB05Rd2hA1jInSDdmnj9VOvIWMQ/6g5TJE6EJuNQXRt3BWD59uXlHJGIHKHuMEXiRNGbt45oXac1S29eStu6bcs/KBEBlIxF4saTT8KoUVD0qcIES6B9vfb+BCUigJKxSNxISTn6qLioPJdHgunqlUh502+diLB+93rSJ6Rz1tNn+R2KSFxSMhaJAx9+CO3awV/+En55gyoNWLx1MYu3LlbnHyI+UDIWiQPffgsrVsCuXeGXpySm0K1JNxyOWT+qa0yR8qZkLBIHFi/23jt3Lr5Oj7QeACzauqgcIhKRUErGInFg5Urvvc1Rw7cU6NSwEwALNy8sh4hEJJSSsUiMcw5++MGbPlYyPrPBmQAs3rq4HKISkVBKxiIxLjMTfvrJm65fv/h67eq2IzmQzOpdq3UTl0g503PGIjFucciBroUbnTwoOZDMP/r/gwZVGpCYoP8aRMqTfuNEYtzpp8MTT0CDBiXXva3HbWUfkIgcRclYJMa1bAm/+53fUYjIseiasYjky87N5tmFz3Lr9FtxzvkdjkjcUDIWiWHOwX33wf/9H+Tmllw/MSGRP378R56c9yQ/7v6x7AMUEUDJWCSmZWXBX/8KN90ECaX4bTez/Eecvtv6XRlHJyJHKBmLxLBZwZ4tGzQ49p3Uoc6o7w3t9F2WkrFIeVEyFolhf/qT936kB67SOKOBl4zV+YdI+VEyFolhmzd771deWfp1Ojf0OrD+dsu3ZRCRiISjZCwSw/bt895/9avSr9O2blsAVu1cxaGcQ2UQlYgUpeeMRWJUXl7BdJ8+pV+vclJlzm52NgmWwLb920irnhb54ESkECVjkRi1Zw/07w8bNhy7T+pwvrj+i7IJSkTCUjIWiVE1a8JHH3nPGotIxaZrxiIxrrSPNBV1KOcQ635aF9FYRCQ8JWORGLVqFfz884mtu2L7Cqo+UJVznz83skGJSFhKxiIxKj0dqlWDpUuPf91WtVuREkhh3U/r2Prz1sgHJyKFKBmLxKBt22D3bm+6bdvjXz8xIZGzGp0FwDdbvolgZCISjpKxSAyaP79gOhA4sW10bNARUOcfIuVByVgkBmVkeO+3337i20hvnA7AvI3zIhCRiByLkrFIDJo713vv3v3Et9GraS8Avsr8SmMbi5QxJWORGOMczAsezHbrduLbObX2qdSsVJOt+7ay+efNkQlORMJSpx8iMWbjRti+HerWhRYtTnw7Zsabl71J0+pNaVi1YcTiE5GjKRmLxJi0NK8rzPXrT7zDjyPOa3leZIISkWPSaWqRGDNuHIwdW3igCBGp2EqVjM1ssJmtMLNVZvaHY9Trama5ZnZp5EIUkePxyivw+OPeABEna++hvYx4dwRDJg85+Y2JSLFKPE1tZgHg38AAYAMw38zecc4tDVPvH8AHZRGoiJQsO7vgsaauXU9+e1WSq/Dykpf5+fDPZO3Lon6V4xz+SURKpTRHxt2AVc65Nc65w8ArwEVh6o0C3gSyIhifiByHhQvhwAGv1626dU9+ewmWQIf6HQD4but3J79BEQmrNMm4CZAZMr8hWJbPzJoAFwPjj7UhMxtuZhlmlrFt2z6/o5cAABciSURBVLbjjVVESvBFcBjic86J3DY7N+wMQMamjMhtVEQKKU0yDnc/ZtEeAJ4A7nTO5R5rQ865Cc65dOdcer169Uobo4iU0pgx3nubNpHbZo+0HgDM3Tg3chsVkUJK82jTBqBpyHwasKlInXTgFfOeo6gLDDGzHOfc1IhEKSIlCu0ka8CAyG23Z1pPAD5b9xm5ebkEEk6ws2sRKVZpjoznA63NrKWZJQNXAO+EVnDOtXTOtXDOtQDeAG5WIhYpXwcPen1RDxwIHTtGbrut67SmZc2W7Dq4i8VbF0duwyKSr8QjY+dcjpmNxLtLOgBMdM59b2Y3BZcf8zqxiJSP1FR47LGy2fY1Z17DoZxD1K0cgbvCROQo5lcH8Onp6S4jQzeEiIhI/DCzBc659KLl6oFLJAY4B5MmwQ8/FL52LCLRQclYJAasXAnXXw89e5ZNMs5zeczdMJeXFr8U+Y2LiAaKEIkF06d77wMHQkIZ/IltGANfGsieQ3sYcMoAGlRtEPmdiMQxHRmLxIBp07z3oUPLZvtmRscG3i3aCzcvLJudiMQxJWORKPfzz/DZZ95wiYMGld1+ujTqAsCCzQvKbicicUrJWCTKffIJHD4M3btDWXZs16WxkrFIWVEyFolyR05RDynjUQ7TG3tPYyzYpGQsEmlKxiJRzjmvw4+yul58RJs6baiSVIXMPZls37+9bHcmEmeUjEWi3NNPw44d0KlT2e7nyHCKVZOrsu6ndWW7M5E4o0ebRGJAamr57Gfa1dOoWakmCaa/40UiSb9RIlEsI8O7eau81E6trUQsUgb0WyUSpXbtgh49oFEjOHSofPe97/C+8t2hSIxTMhaJUpMmQW4utG8PKSnls8/cvFzOff5c6j1cj58P/1w+OxWJA0rGIlHqjju898GDy2+fgYQA+7P3cyDnADPXzCy/HYvEOCVjkSi0L+Qs8WWXle++f9H6FwB8svaT8t2xSAxTMhaJQu+8471Xrgxt2pTvvs9udjYAX6z/onx3LBLDlIxFotDLL3vvDz5Y/vvukdaD5EAyi7YsYueBneUfgEgMUjIWiTK5ubBtmzdUYnmfogZITUqlR1oPHI5ZP84q/wBEYpCSsUiUCQRgzhxYtw4a+DSs8LktzgXgs3Wf+ROASIxRD1wiUappU//2ffnpl9OmTpv8pCwiJ0fJWCSKbN3qnaLu0MHfONrXa0/7eu39DUIkhug0tUgU6dULzjgDfv97vyMRkUhSMhaJErm5sGaNN52e7m8sABmbMrjxnRv519x/+R2KSNRTMhaJEv/4R8G0H3dRF5W1L4v/fvNfXl/6ut+hiEQ9JWORKOAc3HWXN33lld5jTX7r0qgLAN9u+ZY8l+dzNCLRrQL8SotIST79tGB6wgT/4gjVoGoDGlVtxM+Hf2bVzlV+hyMS1ZSMRaLAI4947/ffD1Wr+htLqC6NvaPjhZsX+hyJSHRTMhap4JyD7t2hWTO46Sa/oymse5PuAMxeP9vnSESim5KxSAVnBmPHwtq1UKeO39EUdmTQiNmZSsYiJ0OdfohEiYpw01ZR3Zp0o0/zPvRt3hfnHGbmd0giUUnJWKQCq1sXUlJg1SpITfU7mqNVTqrM57/+3O8wRKJeBfxbW0TAu4N6xw7YtAl2aqRCkZimZCxSATkHt97qTffuDU2a+BvPsWTnZrNg0wLeXfGu36GIRC2dphapgKZMgSVLvOk33vA3lpJk7csi/Zl0aqTUYNedu3TdWOQElOrI2MwGm9kKM1tlZn8Is/xqM1scfH1lZh0jH6pIfNi/Hy65xJt+9FFo2NDfeErSuFpjGlZtyO5Du1m5c6Xf4YhEpRKTsZkFgH8DFwCnAVea2WlFqq0F+jrnzgTuAypIH0Ei0adLl4LpkSP9i6O0zCz/EadP135aQm0RCac0R8bdgFXOuTXOucPAK8BFoRWcc18553YFZ+cAaZENUyQ+/O//wvLl3vScOZCc7G88pdWveT8AZq2f5W8gIlGqNMm4CZAZMr8hWFac/wdMD7fAzIabWYaZZWzbtq30UYrEge3bYfz4gvnu3f2L5Xj1ad4HgM/XfY5zzudoRKJPaZJxuLsxwv62mdm5eMn4znDLnXMTnHPpzrn0evXqlT5KkRiXne2NxrR3L5x9NuTk+B3R8Tm9/unUTq3Nxr0bWfvTWr/DEYk6pUnGG4CmIfNpwKailczsTOC/wEXOuR2RCU8k9jnn9Tk9cyY0aACTJ0Mg4HdUxyfBEujTvA81K9Vk9c7VfocjEnVK82jTfKC1mbUENgJXAFeFVjCzZsBbwLXOuR8iHqVIDLvzTpg40etha+pUb0CIaDRx2ERqVKpBgqn7ApHjVWIyds7lmNlI4AMgAEx0zn1vZjcFl48H/grUAcYFnzHMcc6ll13YIrHBOW8giMREeOst6NHD74hOXK3UWn6HIBK1zK+bLdLT011GRoYv+xapaJYtg/bt/Y4iMvYc2sPug7tpWqNpyZVF4oyZLQh3sKrzSSLl7NAhGDUKVodcWo2VRPz6969T96G6/OXTv/gdikhUUXeYIuUoKwsuvhi++sp7jnjePO80daxoVbsV2XnZfLTmI3LyckhM0H8xIqWhI2ORcrJ4MXTt6iXitDSYMCG2EjFA54adOaXWKWzau4kPVn3gdzgiUUPJWKSM5eV5ibdXL1i/3uvMY/586NzZ78giz8y4vtP1ALy29DWfoxGJHkrGImXswgthxAjYtw+uugo++6ziD/5wMi5q6/WW++HqD8nJi7LeS0R8omQsUsYuvBAaNYJXXoGXXoJKlfyOqGx1qN+BU2qdwpaftzBnwxy/wxGJCkrGIhG0Ywfcfz889FBB2fDh3uAPl18ee9eIwzEzhpw6BMP4Put7v8MRiQp6zlgkAlavhscf93rSOnAAqlWDzEyoUcPvyPyRuTuTSomVqFdFfdCLhCruOWM9dyBygrKzYdo0r1/pLVsKyi+4AEaPhurV/YvNb+rwQ+T4KBmLHIe8PEgIXtxZtgx++cuCZddfD3fcAR06+BNbRZSTl0Pm7kxa1mrpdygiFZqSscgxHD4MCxfCrFnw8cewc6f3WBLAmWfC0KHQuDHce29s3yF9IpZvX07fSX2pVakWy25ZhsXDBXORE6RkLFLEwoUwfjwsWgTffeddAz7CzOtFq359b/699/yJMRqcWvtUcvJyWLFjBSt3rqRNnTZ+hyRSYeluaokrhw/D2rXw+efwwgswdqx3qvmZZwrqbN3qzc+b5yXidu28O6JfeslbdiQRy7ElJiRywakXAPDKkld8jkakYtORsUS9vDzYswd++sl77drlJc0tW+DWWwuu8fbvD5984g1bWFRqKtx4ozfdtSs88QR07Oidiq5du/zaEmuuOuMqJn83mZe/e5m/9v2r3+GIVFhKxlIhzZjhJdQjCTb0dcUV3jO74I0BfOml4RMswLXXQp063nSlSt5p5iZNoFkzaN4cWrSAM86A9JAHDerWhd/9rkybFzcGthpI7dTarNixgqXblnJavdP8DkmkQlIylgrpxhthw4bwy04/vSAZV63qJeJq1aBWLahZ03s1aODdUBWapF980auflFT28YsnMSGRC9tcyPOLnufFRS/yQP8H/A5JpEJSMpYK6YILvHF/jyTXmjW9DjRq1vSS8RHnn+8975tYim9yrVplF68U74azbuD5Rc/zXdZ3fociUmEpGUuFNGFC6eoFAmUbh5y83k178+X1X9KraS+/QxGpsJSMRaRMmRm9m/X2OwyRCk2PNolIuVm2bRkz18z0OwyRCkfJWETKxVeZX3HauNMY8d4IjXMsUoSSsYiUi+5NunNKrVNYs2sNzyx4puQVROKIkrGIlItAQoB7+t0DwL/n/5vs3GyfIxKpOJSMRaTcXHrapTSr0Yzvt33PhAWlvGVeJA4oGYtIuamUWIlHBjwCwH8y/kOey/M5IpGKQclYRMrVRe0uomHVhny/7XveWPqG3+GIVAh6zlhEylVyIJn7zr2PjE0ZDDhlgN/hiFQI5orrYb+Mpaenu4yMDF/2LSIi4gczW+CcSy9artPUIuKr7fu3M/qD0bp+LHFNp6lFxFcj3hvBW8veIicvh8cGPUYgQR2OS/zRkbGI+OqmLjcRsAD/mvcvLn39Ug7nHvY7JJFyp2QsIr4a0GoAUy6fQrXkakxdPpWr3rxKp6wl7igZi4jvLmx7IdOvnk6NlBq8uexNxnw4RglZ4oqSsYhUCL2b9ea1/+81EiyBx+Y8xryN8/wOSaTc6AYuEakwBrYayLtXvsuHqz+kR1oPAHLzcsnal0Wjao18jk6k7JQqGZvZYOCfQAD4r3PuwSLLLbh8CLAf+LVzbmGEYxWRODCk9RCGtB6SPz9zzUyGvDyE81qeR8+0nqQ3TqdTw040rd4U778ekehXYjI2swDwb2AAsAGYb2bvOOeWhlS7AGgdfHUH/hN8FxE5KUu3LSVgAWaumcnMNTPzy6unVKdLoy588j+f5Jc9OfdJAgkBKidVpkpSFSonVSYxIREz47R6p5FWPQ2ATXs3sWL7ivxkbhitarfKXy5S3kpzZNwNWOWcWwNgZq8AFwGhyfgi4AXndec1x8xqmlkj59zmiEcsInHl9p63c23Ha/ls3WfM3TCXKcunsH3/dnYf2s1PB38qVHf0h6PJzgs/NON/hv6Hm9JvAuD9H95n+HvDCy1/eMDD/L7X78umESIlKLE7TDO7FBjsnLshOH8t0N05NzKkznvAg865L4PzHwN3OucyimxrOHDkN6AtsCJSDTkBdYHtPu7fb2p//LY/ntsOar/a72/7mzvn6hUtLM2RcbiLMkUzeGnq4JybAFSIQUzNLCNc/6DxQu2P3/bHc9tB7Vf7K2b7S/No0wagach8GrDpBOqIiIhIGKVJxvOB1mbW0sySgSuAd4rUeQe4zjw9gN26XiwiIlI6JZ6mds7lmNlI4AO8R5smOue+N7ObgsvHA9PwHmtahfdo0/VlF3LEVIjT5T5S++NXPLcd1H61vwLybTxjERER8ag7TBEREZ8pGYuIiPgsppKxmY0ysxVm9r2ZPRRS/kczWxVcNiikvIuZfRdc9q9gt56YWYqZvRosn2tmLULW+R8zWxl8/U95tq80zOz3ZubMrG5IWcy338weNrPlZrbYzKaYWc2QZTHf/tIys8HBz2GVmf3B73hOlJk1NbNPzWxZ8Pf9d8Hy2mb2UfDn85GZ1QpZJ2Lfg4rAzAJm9k2wn4e4ajuAeZ1LvRH8vV9mZj2j+jNwzsXECzgXmAmkBOfrB99PAxYBKUBLYDUQCC6bB/TEe056OnBBsPxmYHxw+grg1eB0bWBN8L1WcLqW320P+Qya4t1o9yNQN57aDwwEEoPT/wD+EU/tL+VnFAi2/xQgOfi5nOZ3XCfYlkbAWcHpasAPwZ/1Q8AfguV/KIvvQUV5AXcALwPvBefjpu3BuJ4HbghOJwM1o/kz8P0DjeAP5jWgf5jyPwJ/DJn/IPjBNwKWh5RfCTwdWic4nYjXW4uF1gkuexq40u+2h8TzBtARWEdBMo6b9ofEdTEwOV7bf4zPpSfwQch8oc8mml/A23j9568AGgXLGgErIv098LutwXjSgI+B8yhIxnHR9mBM1YG1RWOK5s8glk5TtwHOCZ5O+NzMugbLmwCZIfU2BMuaBKeLlhdaxzmXA+wG6hxjW74zs2HARufcoiKL4qL9RfwG7y9ciM/2Fyfa4w8rePqwMzAXaOCCfRwE3+sHq0Xye1ARPAH8L5AXUhYvbQfv7M424Lngqfr/mlkVovgziKrxjM1sJtAwzKK78NpSC+gBdAVeM7NTKL6rzmN14Xki65S5Etr/J7xTtUetFqYs5trvnHs7WOcuIAeYfGS1MPWjsv0REO3xH8XMqgJvArc55/ZY8UMqRvJ74Csz+wWQ5ZxbYGb9SrNKmLKobHuIROAsYJRzbq6Z/RPvtHRxKvxnEFXJ2DnXv7hlZvZb4C3nnVOYZ2Z5eB2CF9dV54bgdNFyQtbZYGaJQA1gZ7C8X5F1PjvxFh2f4tpvZmfgXQdZFPzPKA1YaGbdiIP2HxG8oeoXwPnB7wHEUPsjIKa6rTWzJLxEPNk591aweKsFR4wzs0ZAVrA8kt8Dv/UGhpnZEKASUN3MXiI+2n7EBmCDc25ucP4NvGQcvZ+B3+f+I3gN4Sbg3uB0G7zTCwacTuEL92souHA/H+9I+siF+yHB8lsofOH+teB0bbzrFLWCr7VAbb/bHuazWEfBNeO4aD8wGG9Yz3pFyuOi/aX8jBKD7W9JwQ1cp/sd1wm2xYAXgCeKlD9M4Rt4Hor096AivfD+ODxyzTje2v4F0DY4fXew/VH7Gfj+gUbwB5MMvAQsARYC54Usuwvv7rkVBO+UC5anB+uvBp6ioEeySsDreN17zgNOCVnnN8HyVcD1fre7mM9iHcFkHC/tD8aTCXwbfI2Pp/Yfx+c0BO/O49V4p/d9j+kE23E23inDxSE/8yF41/Q+BlYG32uHrBOx70FFeVE4Gcdb2zsBGcHvwFS8P5Cj9jNQd5giIiI+i6W7qUVERKKSkrGIiIjPlIxFRER8pmQsIiLiMyVjERERnykZi5QB80bOKum1Llh3kpltKGGT5cLM7g7GFpEOgY5srxT1+gX32y8S+xWJNlHVA5dIFOlZZH4KXqcDd4eUHSq3aESkQlMyFikDzrk5ofNmdgjYXrT8ZJlZinNOSV0kyuk0tUgFYWadzewLM9sfHBz9piLLfx08ldvHzF43s5/wRio6sryvmX1sZnvNbJ+ZfWBmHYpsY5CZzTaz3Wb2c3Cg9b+GCaelmb0frPOjmf3VzBKKbKutmU0xs5/M7ICZzTGzwaVoZz0ze9nM9gTXfQFvLFqRuKVkLFIxVMcbKP4l4CK8/nL/Y2bnhqk7Ga9f7EsJjlRjZkPxuv/7GbgGuAqoBnxhZk2DdU4B3sHrLvVyYBjwGFAlzD6mAJ8Av8TravAe4H+OLDSzxsCXeONnjwQuA34C3jezC0po61t4A3r8KRhHDvBkCeuIxDSdphapGKoBNzvnPgUws1l4Q2JeCXxapO4bzrn/LVL2T+Bz59xFRwrM7FO8DvFHA7fhDTmXDPzWObcnWO2TYuJ51Dn3XHB6ppmdF4zlSNkdeH0B93TOrQrubxreYB1/o2A86ULMbABev9JXOudeCRZ/YGbTKTx6jkhc0ZGxSMWw/0giBgheB14JNAtTd0rojJm1BloBk80s8cgL2A98DfQJVv0WyAZeMbNLzaw+xXu/yPySIrH0AeYcScTBmHOB/wM6mVn1YrbbE8jFG/ow1Cth6orEDSVjkYphV5iyQ3gjxxS1ucj8kaT6LF6yDX39Am8kG4KJcxDe7/2LwBYzm2tmfcPso+i4rUVjqR0mDoAteEPR1QqzDKARsMs5l12kfGsx9UXigk5Ti0Sfos/t7gi+/xGYGab+4fwVvaPvT80sBW+Q+nvxrvO2cM5tP44YdgINw5Q3DMZX3CDsm4FaZpZUJCE3OI59i8QcJWOR6LcC76as051zD5ZmheBp8E/MrCrwNt6A68eTjD8Hbgsm8XUAZhbAuyHrG+fc3mLW+xoIAJdQ+NT0Fcexb5GYo2QsEuWcc87MbgHeNrNk4DW8xNoA6AWsd849FnxUqg8wDcgE6uIdTW/CuyZ8PB4Hfg18ZGZjgT3AzUAbYOgxYv3IzL4EnjazunjXxS8HOhS3jkg80DVjkRjgnJuGl2irAP8FPgAewjtt/HWw2qLg8geAD4Gn8B6ROs85d+A497cJ767o74H/AG/gXUce6pybUcLqv8L7g+AB4FW8g4KRx7N/kVhjzpXYbayIiIiUIR0Zi4iI+EzJWERExGdKxiIiIj5TMhYREfGZkrGIiIjPlIxFRER8pmQsIiLiMyVjERERn/3/TNIHTSLsE9YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 使用matplotlib 绘制精度和召回 相对于阀值的函数图\n",
    "def plot_precision_recall_va_threshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], 'b--', label= 'Precision', linewidth = 2)\n",
    "    plt.plot(thresholds, recalls[:-1], 'g--', label='Recall', linewidth=2)\n",
    "    plt.xlabel('Threshold', fontsize=16)\n",
    "    plt.legend(loc='upper left', fontsize=16)\n",
    "    plt.ylim([0,1])\n",
    "plt.figure(figsize=(8, 4))\n",
    "plot_precision_recall_va_threshold(precisions, recalls, thresholds)\n",
    "plt.xlim([-70000, 70000])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 通过选择阀值来实现最佳的精度/召回率权衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 目标设定为90%的精度 ，阀值大概在7500左右,设置阀值\n",
    "y_train_pred_90= (y_scores > 3000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8078138465958058"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision_score(y_train_8, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4806016065629807"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_8, y_train_pred_90)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 21484 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 22238 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 21484 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 22238 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 31934 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 24230 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 31934 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 24230 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAAF8CAYAAAAuF9n2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3debxd873/8dcno4QQJDEEjTGtIagjOqCGGmKKVluiaijVXLO6RXvvLbf91c9wW1xyrxK3tNqqW6miQtGqmiXmGNKIIREkQYXEkMT398c3+e1z4iQ55+Sctfbwej4e+7HXd+211/lYVe+91vqu7zdSSkiSpMbQrewCJElScQx+SZIaiMEvSVIDMfglSWogBr8kSQ3E4JckqYEUGvwR8T8RMTMinlrK5xER/xkRUyLiiYj4dJH1SZJU74o+478K2HsZn48ANl30Ohb47wJqkiSpYRQa/Cmlu4E3l7HJSOAXKXsA6B8R6xRTnSRJ9a/a7vEPBqY1a09ftE6SJHWCHmUXsIRoZV2rYwpHxLHk2wH07Lnadv36bfL/P+vbF9Zaq0vqkySpKkycOHF2Smlge79XbcE/HVi/WXs9YEZrG6aULgcuB2hqakoTJkxo8fn55+f3I4+E3/4WbroJ5s6F8eNh1VU7v3BJkooUES915HvVFvw3AidExLXADsDbKaVXO7KjM87I7z/4AXzwQV4+4QSYPdvglyQ1rkKDPyJ+A+wCDIiI6cBZQE+AlNJlwC3APsAUYB5w1Ir+zQ8/hP32g699Lb9/+cvwH/8B2223onuWJKn2FBr8KaVRy/k8Acd3xt865ph8dv/jH8Pmm+d1558PCxbAddcZ/JKkxlRtl/o7zRVXfHzdCy/APffk13e+YwdASVLjqbbH+brUJz9ZWb7//vLqkCSpLA0V/CedVFn+70VjAs6eDaeeChtsAD/9KTz/fDm1SZJUhIYK/gjYe9GAwf37w6WXwkYbwUUXwbRpcNppcM455dYoSVJXaqjgB/j5z+HOO2HDDeGUU+CddyqfRSw9+N99Fy68EDbdFA4+GObPh/ffL6ZmSZI6S8MF/9prw7bbwiOPwPbbw7hx8N57+X3WrBzwX/ta7vh3yCFwxBGQEnz0Ue4QOGVKfiqgVy/o0yeH/8svw8CBcPHFZf/TSZK0bA0X/ACrr54v8996K3zpS7DSSvn9kktgk03gf/8XZs7MI/7ddVf+QbDqqq0H+7x58PTTua/AKadA796w5Zbwxhstt3vvvdZrefLJPKKgJElFqNvH+ZZns80+vq5HK0fj8MNhjTXy8kkn5TEB7rgDFi7M61ZbDT71qcr2H34IkyblqwVXXQW/+lX+AfHII/nzr3417++yy2DOHDjqKJg4Mb/vsQeMHJmvJvTokf/GvHnQr1+n/qNLkhpY5DFzaltrY/V3REr5nn/v3vm9T58c5KuvvvzvPvUU7LZbvjoAOfynToX99//4tquskq8I3H8/7LLLxz8fMgQmT4YLLoB/+ZeW66+5Jt+i6NUrr1u4ELp3b+c/qCSp5kXExJRSU3u/15CX+pcmIl/S790bBgyAlVduW+hDvrw/c2bu9Pf738O66+Yz+COPhOuvhxNPzNucdx488AD07Ak77gjf/e7H9zV2bH5/8cWW6198MX/nn/8Z/v73PPpgjx657sWv115bgQMgSap7nvFXidmz4dFHoamp8mNj/nz4wx/gscfy0MOQfyicd17uF7C0WwAp5b4DZ58Np5+erxB08yeeJNWVjp7xG/w17IEH8hWKBx+E//zP/APhT3+Cvn1zH4Pp0yvbfuMbcPXV+aqAJKn2GfwNGPxLmjkzn9n37ZuDfty4lp/37w9vvVVObZKkzuU9fjFoUO6b0Ldv7lfw3nt5QqLFrr463wL47Gfzmf+0aeXVKkkqh8Ffx1ZaCT7/+XzPf+5cGDo0jz74wAP58w02yD8AhgzJ7dmz8/gGn/lMHsRoccfBT38a/vznvM1NN+UBkPbZB557Lncy/NWv8syHkqTq17DP8Teavn3zcMN77JH7Ajz6aOWzTTaBGTPyj4MTT8zrHnyw8vmjj8K99+bHFa+/Pn//scdg/PjKNp/7XN5m4kTYaad8S6F372L+2SRJbWfwN5Bu3WDUqPz66COYMCEPGLTnnvnxw5TyFYIePfIgQ/3756GI99wzPxkAedyBq6/++L6/+EV4/PH8VALkqw2/+U3+7uIBkCRJ5bNznzpk9uzch2C99SpPCixc2Proh9/+dh6p8NJL83f22AO22abYeiWp3ti5T4UaMADWX7/l44Hdu+erBsccU1m31Va5z8CsWXneg9NPz30EFg+WdMopeehiyN+VJHUtz/jVZV5/PXcK/NKX8v3+pQ0i9Lvf5R8DG2+c2z/5CYwYkTsjOvCQJLXOM35VnbXWyv0JVlopn+GnlF+/+x1svXW+QnD99XDQQfDXv1a+d9ppeTKk7t3zFMiQOxjOmZP7JkiSOs7gV+EOOig/FbBgAXz5y3ndIYfk2Q+b69MnPz747rv5kcLVVss/BiLynAXNRyaUJLWNwa+q0KcPXHxx5arA44/D5ZfDL38Jt9328e3vvTf3MUgJbrghP044eXLxdUtSrTH4VZWGDYPDDsvLBx2UA37BgspAQpCvELzxRp618P77c5+AXr3y0wWSpNb5HL9qRvfusOuuLXv/pwQ/+1mlPX9+fqTwyCPh5z8vvERJqnqe8aumRcAzz8Dbb1fW9eiROwPOmgVXXumcBJLUnMGvurDqqvns/6WX8v3+O+6AefPymAKL5yRY/Go+1LAkNRqDX3Vlgw3yo4EDBsBDD7W+zZgx+f0//gO22y4/OeBtAUmNwuBX3frqV/NVgD/+EdZeO09UBHDccTB2LFxzDTzyCBxwAHzzm/lqwNFH520WLoRf/CJ3GpSkeuLIfWpIU6dWRgpsbt114ZVX8kBB/frl2wUAl1ySnzLo37/YOiVpaRy5T2qHjTaqjBmQEjz8MAwfDiefnD+PqIQ+5OmKV18d/uVfKuvq4DezpAZk8Evk6YQffDBPIgSVIYbHjq1ss/HG8I9/5PUnnZTnEVjcYXDLLcupW5Lay+CXluHoo3PQz5iRJxu68EL41rfypf/mJk1qOSuhJFUrg19qg3XWgQsuyCMDXnJJHjvgjTdaTi40fHh+P/jgPARx81sFklQtDH6pnfr0gU9+EtZYA3beOY8WePXV8IUvwDvvwK23wvvvw8or59sAn/1sy6GGJalM9uqXOlFK+d5/a+bOrTxSKEkryl79UhVY3Cnwvfdg3DjYZpu8/sgjc+h/8AGMHp0fF5SkMnjGLxVg/vw8u+C66+YnAyAPErS0qwOStDye8UtVrGfP3DdgcehDnm3w5psdD0BSsQx+qUBLhvz++8PQoeXUIqkxGfxSwT76CH7yk0r79tthzpw8nfDuu8Pf/15ebZLqn8EvFSwCvvOdfPb/+ON5/P+XXsr3/P/8Z9hss7xNv34wa1bZ1UqqNz3KLkBqZMOG5fd+/XJHv+a9/d99FwYNyh0De/j/VEmdxDN+qQoMGZLP+FPKcwYMGpTXX3VV/gHw7/+ehwWWpBVl8EtVZvhweP11mD4d9t0XJkzIPwa23BJGjMhXACSpowx+qUoNHgwDBsBKK8H48Xndrbfm+QKuvrrc2iTVLoNfqnI77vjxCX+OPBK+8pVSypFU4wx+qQb06ZPv/z//fGXdJZfkdRH5NXFiefVJqh0Gv1RDNtooh/0TT+Sx/2+9tfJZU1PuB/D+++XVJ6n6GfxSDdpqK1htNdhzTzjuuMr6SZPy1YH77iuvNknVzeCXalj37jBmTMvR/vbbDx5+GF59Nd8C+Nd/zRMESRIY/FJd2GSTfAvgL3+BL38ZTj4ZRo3Kn/34x3mSoMWjBUpqbAa/VEd22QWOOiovH3JIy88uvDCPDvjss4WXJamKGPxSnRo9Op/h33JLZV3v3vCzn+XHA486Kl8VaD5MsKT65wjgUp0bMSL/ALjhhny2f/rpOfT/8heYNg2uvTY/CdC7d9mVSiqCZ/xSgzjwQDjzzHy5/6CDcugvttJKMHRoHiZYUn0rPPgjYu+IeC4ipkTEma18vlpE3BQRj0fEpIg4qugapXp3wAH5KsCBB1bWTZ4Mc+bk5S9+EVZZJXcK/OQn7RQo1ZNCgz8iugNjgBHA5sCoiNh8ic2OB55OKW0N7AL8JCJ6FVmn1Ch+//s8GNCmm8LGG+fpf++/Pz8GOHdufgzwuefyVYKnniq7Wkmdoeh7/MOBKSmlqQARcS0wEni62TYJ6BcRAawCvAn4FLLURbbaKp/tL/bWW/my/3bbwXnntdzuH//IAwdJql1FX+ofDDS7s8j0ReuauxT4FDADeBI4OaVkv2OpIKuvDpdeCueemy/xX3llXr/rrvny/z33VOYHWLiw3FoltV/RwR+trFvy7uFewGPAusA2wKURserHdhRxbERMiIgJs2bN6vxKJQHwzW/mHwB33glTpsBOO1U+69EDTjnF+QGkWlJ08E8H1m/WXo98Zt/cUcC4lE0BXgA+ueSOUkqXp5SaUkpNAwcO7LKCJWUR+RbASy+1XH/xxZXZAyVVv6KD/2Fg04jYcFGHvUOAG5fY5mVgd4CIWAsYCkwttEpJS7XBBjnk99ijsm7GjPzDQFL1KzT4U0oLgBOA24BngOtSSpMiYnREjF602Y+Az0XEk8CdwBkppdlF1ilp+f70p/wDYP58WGMNuOuuHP7z5pVdmaRlKXzkvpTSLcAtS6y7rNnyDGDPouuS1DE9Fv1X5KST8vvKK+f3KVPyI4KSqosj90laYT16wPbbt1y3ySZw773l1CNp6Qx+SZ3iyivhxRdhiy0q6374w3wrQFL1MPgldZpPfCKP8DdmTG6PGpWH/R09Ot//nzSp3PokGfySusBxx+WOf0ceCfvtl6cCBthyy/wD4Mc/LrU8qaEZ/JK61E035bkAmvvXf4XTTiunHqnRGfySulREngvgo4/grLMq6//pn+Ddd3PP/wMPzJMCSep6Br+kQkTA2WfnWwAvvACDB+dH/qZOhT/8Ic8D4ARAUtcz+CUVbsiQPMzv4CWm6Jozx/CXuprBL6k0AwfmKwBz5lTWzZkD991XXk1SvSt85D5JWlK/fvkHwFprwb77wrBhuU9AN09NpE5n8EuqGtOnw2uv5dsA3bvDoEG5D8DiYYAlrTh/T0uqGj17wvrrw3e/m9szZ+ZOf/vvX25dUj0x+CVVnZ/+FIYOrbRvvhnOOKO8eqR6YvBLqkrPPpvv8y92/vlw7rnl1SPVC+/xS6paEbBgARxwALz1FpxyStkVSbXPM35JVa1793yp/4Yb4O238xDAEfDrX5ddmVSbDH5JVS8i9/B/+ul89g/w9a/n9TNmlFubVGsMfkk1Y9ddKzP9LTZ4MFx5ZR4HQNLyGfySasqxx+ZOf83v9w8aZPBLbWXwS6o5EXDhhfD883DMMbDNNvDSSzB+fNmVSdXP4JdUszbaCK64Ig/1O20a7LNP/lHw7rtlVyZVL4NfUs3r1QtOPrnS7tcPvvEN+OCD8mqSqpXBL6kuPPoonHRSpX3NNbDSSvDhh+XVJFUjg19S3bj4YnjqqZbrfv97mD+/nHqkamTwS6orW2yRe/iPGwfPPQcHHwybbZbv/V97bdnVSeUz+CXVpS99KQf+I4/Aiy/mdaNGwY47llqWVDqDX1Jd23ZbOOywSvvee6F///Lqkcpm8EuqaxHwy1/myX4We/vt/JIakcEvqSF07155vO+ccxzpT43LaXklNYxevSqBf9ttsPfeefn11/Owv1Ij8IxfUkP63/+tLK+1Flx3XXm1SEUy+CU1pDFj8uX/xQ4+uLxapCIZ/JIaUu/eucPfnXdW1p15Znn1SEUx+CU1tN12y+/9++fpfqV6Z/BLanhz5+ZH/s45B84+Ow/6I9Urg19Sw+vbF/bbD846K1/63267/Py/z/qrHhn8krTIsGFwzz2VtiP8qR4Z/JK0yMEH5+f8jzmmsu7ii8urR+oKBr8kLeGKKyrLp5xSGfFPqgcGvyS14rnn8vsvfpGf93eIX9ULg1+SWrHZZvnxvlGj4MknoVs3p/RVfTD4JWkpIqBHD/jmN3P73nvhggvKrUlaUQa/JC3HxImV5dNPh9/9zkv/ql0GvyQtR7du8OyzlfZXv5rXOdKfapHBL0ltMHQo/OlPlfYdd+Twl2qN/9pKUhvtsUe+xP/667mjX0rwm9/AW2+VXZnUdga/JLXToEHQs2cO/0MPhTXWgH//97KrktrG4JekDujWLXf0W+zss/NTAN73V7Uz+CWpg0aOhDfeaLnu058upxaprQx+SVoBa6yR7/Wvs05uv/++Z/2qbga/JHWCV17J7z/7Wb7k73P+qlYGvyR1gsVhv/PO+Zn/bt3g/vvLrkr6OINfkjrRK6/A5pvn5c99Dm6/vdx6pCUZ/JLUidZbD666qtLec0+4666yqpE+zuCXpE52xBHwyCOV9q67wtSp5dUjNWfwS1IX2HbbyhC/O+wA06eXW4+0mMEvSV1kjz3gvvtg991zp7/jj3daX5WvR9F/MCL2Bi4GugNjU0rntrLNLsBFQE9gdkrpC4UWKUmd5LOfza/XXssT+0yeDN27w3e+U3ZlalSFnvFHRHdgDDAC2BwYFRGbL7FNf+C/gANSSlsAXy2yRknqCn375tAHOO00eOihcutR4yr6Uv9wYEpKaWpK6UPgWmDkEtscCoxLKb0MkFKaWXCNktTpVl0V3nmn0t5hB/jjH8urR42r6OAfDExr1p6+aF1zmwGrR8RdETExIg5vbUcRcWxETIiICbNmzeqiciWp86yyCvz+95X2fvvB/Pnl1aPGVHTwRyvrlhzYsgewHbAvsBfwbxGx2ce+lNLlKaWmlFLTwIEDO79SSeoCBx4IL7yQl3/4Q5gzp9x61HiKDv7pwPrN2usBM1rZ5taU0tyU0mzgbmDrguqTpC43ZEieyOeMM+Cpp2DYMBg3ruyq1CiKDv6HgU0jYsOI6AUcAty4xDZ/AHaKiB4R0RfYAXim4DolqUtFQM+esMsu8OSTcNBB8OabZVelRlBo8KeUFgAnALeRw/y6lNKkiBgdEaMXbfMMcCvwBPAQ+ZG/p4qsU5KKEAGTJlXaa64Jh7faq0nqPJHqYO7IpqamNGHChLLLkKQOueACOP30SrtXL3j//fzDQFqaiJiYUmpq7/ccuU+SSvbd78KHH1baJ54Is2eXV4/qm8EvSVWgZ09ICb73vTy878CBMGIE3HNP2ZWp3hQ+ZK8kaenOOSe/b7xxntHv1lth7tw88p/UGTzjl6QqdNFFleWVV64M9yutKINfkqrQ/vvDMcdU2kOHlleL6ovBL0lV6oor8uh+i9nLX53B4JekKvZv/wYjR8Jqq8GMGXmin8VD/kodYec+SapyN9wAM2fmnv5Tp8Imm+T1dTAMi0rgGb8k1YBBg/Kl/uZj+nvfXx1h8EtSDfnudyvLkyfncf6l9jD4JanGLFhQWR42rLw6VJsMfkmqMd27w/nnV9qXX15eLao9Br8k1aB//mfYYot8z//oo+G552DatLKrUi2wV78k1aAIeGrRhOXvvAOnnQZ//KM9/bV8nvFLUo2bOTOHPuQrAdKyGPySVOM23riy/JOfeNavZTP4JakONJ++t1s3WLiwvFpU3Qx+SaoDn/98nsVvsTFjyqtF1c3gl6Q6MWdOfv/e9/IPAak19uqXpDrRrVvL+/tXXpmn9vWev5prU/BHxOEd2PfclNL1HfieJGkFvfRSDn3IY/o/91y59ah6tPWM/xLgYqA9s0EfARj8klSCT3yisjx5Mnz0Ub4iILU1+OeklH7Qnh1HxJHtL0eS1Fk++AB6987LffrkttTW338duUPkXSVJKlGvXrDXXnn5ww/zaH+SF34kqY7deit8/et5efTocmtRdTD4JanOXXMNPPEEXHJJ7uH/7W/D+++XXZXKYvBLUgPYaivo0QMeeCBP49unT9kVqSxt7dy3UkTs2Y79Bu17AkCSVIDzz68s33kn7L57ebWoHG0N/j8Ch7Rz379t5/aSpC42blzlsb4vfhEuvRSOP77cmlSstgb/kA7se+XlbyJJKlIEXHEFfOtbuX3CCXDUUdC3b7l1qThtDf6hwI60/fJ9AH/uUEWSpC51zDHwhS/AZptBUxNMnAg77VR2VSpKW4N/YUppant2HOETo5JUrTbdFO64A4YMgbXXhrlzW87up/rlAD6S1KB23x023hieegpWWSWP76/65+N8ktTA5s+Hz3wmLw8Z4kx+jcDgl6QG1rMnXHVVpe1EPvXP/4klqcEdcQTsu2+lve665dWirtfWzn1rRsSv27FfO/ZJUg25+ebKJD6vvgoTJuQe/6o/bQ3+fTqw74s78B1JUkk+/DDP6Pe5z8F225VdjbpKm4I/pfTXri5EklSunj1z574FC2DmTPjRj/LIfqov3uOXJLUwYwYMHw5jxni/vx4Z/JKkFjbYAF5+OS+/+mq+9//uu+XWpM5j8EuSPub991u2+/WDu+4qpRR1MoNfkvQxvXvDRx/ls//Fzj67tHLUiQx+SVKrIvIwvmPH5vZf/pLH9B8/vty6tGIMfknSMh19dO7tHwGvvQb77AMHHFB2Veoog1+S1GY/+1l+v+kmOO+8cmtRxxj8kqQ2O/fcyvKZZ8K4ceXVoo4x+CVJbdatGzz7bKV90EHl1aKOMfglSe0ydGjLR/v+8pfSSlEHGPySpHb7whfy+9Zbw667wt13w2OPlVuT2sbglyR1SEo57OfNy53+tt0WDjyw7Kq0PAa/JGmFLFgAv140cfsf/gAXXFBuPVo2g1+StEJWXbXlWP6nnw5PPllePVo2g1+StMJWXhmmTau0hw3LtwJUfQx+SVKnWG89uOOOvLzaajB5crn1qHUGvySp0+y+O9x/P7z1Fqy/vj39q1HhwR8Re0fEcxExJSLOXMZ220fEwoj4SpH1SZJWzGc+ky/zP/xw7ukfUXZFaq7Q4I+I7sAYYASwOTAqIjZfynbnAbcVWZ8kqXNEwN57V9pjxpRXi1oq+ox/ODAlpTQ1pfQhcC0wspXtTgSuB2YWWZwkqXNE5Of7FzvhBDjrrPLqUUXRwT8YaNbvk+mL1v1/ETEY+BJw2bJ2FBHHRsSEiJgwa9asTi9UkrRiIuDttyvtH/7QM/9qUHTwt3anZ8kHPi4CzkgpLVzWjlJKl6eUmlJKTQMHDuy0AiVJnWfVVWHq1Lw8cCBsuWW59aj44J8OrN+svR4wY4ltmoBrI+JF4CvAf0WEg0BKUo3acEN4+WUYMSKP8f/zn+erAT7nX45IBR75iOgBTAZ2B14BHgYOTSlNWsr2VwE3p5R+t6z9NjU1pQkTJnRytZKkzvbqq7DuupW24d9xETExpdTU3u8VesafUloAnEDurf8McF1KaVJEjI6I0UXWIkkq3jrrwPDhlXYEzJ1bXj2NqNAz/q7iGb8k1Y4FC6Bnz5br6iCKClcTZ/ySJPXokYP+hBMq6844o7x6Gk2PsguQJDWmSy6BZ5+F/feHr32t7Goah8EvSSrN7bfDBx9A795w9NF5Yp+//a3squqbl/olSaXq3Rseegh++1u45558JUBdx+CXJJVu++0rvftPOglmzy63nnpm8EuSShfRcgrfgQPzYD/qfAa/JKkqbL01XHpppX333fDGG+XVU68MfklS1Tj+eFi4EPr0yUP7rrFG2RXVH4NfklRVunXLU/oedlie3e/HPy67ovri43ySpKo0bx6svnpefvJJuPbacuupF57xS5Kq0qqrQt++efm3v81n/1pxBr8kqWpNn15Z7t+/ZVsdY/BLkqrW6qu3HMd//fVz5z91nMEvSapq554LZ5+dl8eOzUP8quPs3CdJqnpnnQUHHQT9+uX7/r/+dV5/6KHl1lWLDH5JUk3Ycsv8/vTTMG4cXH89/P3v+UeB2s5L/ZKkmrLuujn0Id8CePPNUsupOQa/JKmm9O8P8+dX2muuCTNnlldPrTH4JUk1p0cP+OlPK+211oJnny2vnlpi8EuSatKpp8Jll+Xliy6CAQPKradWGPySpJr17W/DK6/kKXwHDIBXX4X99y+7qupmr35JUk1bd938eugh2GGHvG7tteG118qtq1p5xi9Jqgvbb1955O/11+HEE8utp1oZ/JKkuhABEyZU2pdemmf1U0sGvySpbvTu3XJI32HD4L33yqunGhn8kqS60qsXjB+fl484Ah57rNx6qo2d+yRJdWfvveHRR+ETn4DVVoMFC/Kz//KMX5JUp7bZJk/r+/LL0LMn/OlPZVdUHQx+SVLdWrgQvv/9vLzXXvC3v5VbTzUw+CVJdat7dzjnnEp7552dzc/glyTVtSFD4M47K+0f/rAy1G8jMvglSXVvt91azuD3T/9UXi1lM/glSQ1h4EB4/nk4/nh4+OGyqymPwS9JahgbbZRH9GtqgltugS22KLui4hn8kqSGkxI8+yw8/TSMGlV2NcUy+CVJDScCTjstL197Lfz85+XWUySDX5LUkN55p7L8zW/CwQeXV0uRDH5JUkNaZRV48MFK+7rr4K23yqunKAa/JKlhDR+ex/EH2G8/eOaZcuspglMWSJIaWvfuMHduvu/fpw/ccEN+5v/YY8uurGsY/JKkhte3b34/80y45hp45RUYMAC+/OVy6+oKXuqXJGmRI47IoQ9w0EHw+c+XW09XMPglSVrkU59qOarffffBlluWV09XMPglSWqmqQk++gh22im3J03Kz/rXC4NfkqQlRMDdd+fl730vdwCsF3bukyRpKWbPhtdey2P6n3ce7LUXbLNN2VWtGM/4JUlaijXXzKH/wQdw++2w7bbw61+XXdWKMfglSVqO116DO+/My1//eh7lr1YZ/JIkLccnPtGyg18tj+tv8EuS1AYHHwx//Wulvdlm5dWyIgx+SZLaaKedoFevvPz3v+ehfmuNwS9JUhtF5I5+N9yQn/VfeeWyK2o/g1+SpHYaOTL/CJg9O7/PmlV2RW1n8EuS1AEffAD77puXBw0qt5b2MPglSeqA3r1ht90q7e9/v7xa2sPglySpg/7v/225/K1vlVdLWxUe/BGxd2+iCCgAAAl6SURBVEQ8FxFTIuLMVj7/ekQ8seh1X0RsXXSNkiS11cyZleWxY+Hmm8urpS0KDf6I6A6MAUYAmwOjImLzJTZ7AfhCSmkY8CPg8iJrlCSpPQYOzJ38AE4+Gd55p9x6lqfoSXqGA1NSSlMBIuJaYCTw9OINUkr3Ndv+AWC9QiuUJKmd1lwTXn0V3n0XNtkExo/P9/979y67so8rOvgHA9OatacDOyxj+6OB8V1akSRJnWDttfP7W2/BYYfBm29CSuXW1Jqi7/FHK+taPSwRsSs5+M9YyufHRsSEiJgwq5YeoJQk1bXjj8+hD3DsseXW0pqig386sH6z9nrAjCU3iohhwFhgZErpjdZ2lFK6PKXUlFJqGjhwYJcUK0lSe11zTWX5iivg4ovLq6U1RQf/w8CmEbFhRPQCDgFubL5BRGwAjAO+kVKaXHB9kiStkG7d8v3+xU45Bfbeu7x6llRo8KeUFgAnALcBzwDXpZQmRcToiBi9aLMfAGsC/xURj0XEhCJrlCRpRa29Njz0UP4RAHDbbdXT2z9SNfY8aKempqY0YYK/DyRJ1eXdd6Ffv/ysf2fflY6IiSmlpvZ+z5H7JEnqIqusknv2DxwITz2VJ/Qp+zzV4JckqQBbbZXft98+T+lbFoNfkqQCXHhhZbl7d1iwoJw6DH5JkgpwyimVzn4APXvC++8XX4fBL0lSQRYuhGOOqbT79Cm+hqKH7JUkqaFdcQXstBM8/jicemrxf9/glySpYIcfXllOCWbNgkGDivnbXuqXJKkkd98NW28Na60FL71UzN80+CVJKsmwYfDkk3l5yBCYPbvr/6bBL0lSSfr3h1/9qtIeOBAuv7xr/6bBL0lSiQ49NHf4W+zb34a33uq6v2fwS5JUsmOOgenT8/Kpp8LEiV33t+zVL0lSFRg8GH75y3y2/8Uvdt3f8YxfkqQqcdhhcOKJ8MYbMHIk3H575/8Nz/glSaoyAwbk9xtvhPnzoUcnprVn/JIkVZnx4yvLPXt27r4NfkmSqszee+fpexc7//zO27fBL0lSFXrwwcryGWfAvHmds1+DX5KkKhSRx/AHWGedzuvoZ+c+SZKq1IAB8Ne/5qF9+/fvnH16xi9JUhXbeecc+v/4B/zoR3k2vxXhGb8kSVVu2jTYbTeYMQPWXBOOO67j+/KMX5KkKrf++vk+/7x5cPzxK7Yvg1+SpBpw1FGV5a98peP7MfglSaoBRx2V7/cDXH99x/dj8EuSVCOaj+jXUQa/JEk1om/f3LnvG9/o+D4MfkmSasjjj8Ouu3b8+wa/JEk1ZPDglh392svglySpgRj8kiQ1EINfkqQGYvBLktRADH5JkhqIwS9JUgMx+CVJaiAGvyRJDcTglySpgRj8kiQ1EINfkqQGYvBLktRADH5JkhqIwS9JUgMx+CVJaiAGvyRJDcTglySpgRj8kiQ1EINfkqQGYvBLktRADH5JkhqIwS9JUgMx+CVJaiAGvyRJDcTglySpgRj8kiQ1EINfkqQGUnjwR8TeEfFcREyJiDNb+Twi4j8Xff5ERHy66BolSapXhQZ/RHQHxgAjgM2BURGx+RKbjQA2XfQ6FvjvImuUJKmeFX3GPxyYklKamlL6ELgWGLnENiOBX6TsAaB/RKxTcJ2SJNWlooN/MDCtWXv6onXt3UaSJHVAj4L/XrSyLnVgGyLiWPKtAIAPIuKpFaxNyzYAmF12EQ3A49z1PMZdz2NcjKEd+VLRwT8dWL9Zez1gRge2IaV0OXA5QERMSCk1dW6pas5jXAyPc9fzGHc9j3ExImJCR75X9KX+h4FNI2LDiOgFHALcuMQ2NwKHL+rd/xng7ZTSqwXXKUlSXSr0jD+ltCAiTgBuA7oD/5NSmhQRoxd9fhlwC7APMAWYBxxVZI2SJNWzoi/1k1K6hRzuzddd1mw5Ace3c7eXd0JpWjaPcTE8zl3PY9z1PMbF6NBxjpyzkiSpEThkryRJDaSmgt/hfrteG47x1xcd2yci4r6I2LqMOmvZ8o5xs+22j4iFEfGVIuurF205zhGxS0Q8FhGTIuKvRddY69rw34vVIuKmiHh80TG2z1Y7RcT/RMTMpT2y3qHcSynVxIvcGfB5YCOgF/A4sPkS2+wDjCePBfAZ4MGy666lVxuP8eeA1Rctj/AYd/4xbrbdn8n9Yb5Sdt219mrjv8v9gaeBDRa1B5Vddy292niMvw+ct2h5IPAm0Kvs2mvpBewMfBp4aimftzv3aumM3+F+u95yj3FK6b6U0luLmg+Qx1lQ27Xl32OAE4HrgZlFFldH2nKcDwXGpZReBkgpeazbpy3HOAH9IiKAVcjBv6DYMmtbSulu8nFbmnbnXi0Fv8P9dr32Hr+jyb801XbLPcYRMRj4EnAZ6qi2/Lu8GbB6RNwVERMj4vDCqqsPbTnGlwKfIg/C9iRwckrpo2LKaxjtzr3CH+dbAZ023K+Wqs3HLyJ2JQf/jl1aUf1pyzG+CDgjpbQwnyipA9pynHsA2wG7A32A+yPigZTS5K4urk605RjvBTwG7AZsDNweEX9LKc3p6uIaSLtzr5aCv9OG+9VSten4RcQwYCwwIqX0RkG11Yu2HOMm4NpFoT8A2CciFqSUbiimxLrQ1v9ezE4pzQXmRsTdwNaAwd82bTnGRwHnpnwzekpEvAB8EniomBIbQrtzr5Yu9Tvcb9db7jGOiA2AccA3PDPqkOUe45TShimlISmlIcDvgOMM/XZry38v/gDsFBE9IqIvsAPwTMF11rK2HOOXyVdUiIi1yJPKTC20yvrX7tyrmTP+5HC/Xa6Nx/gHwJrAfy06I12QnIyjzdp4jLWC2nKcU0rPRMStwBPAR8DYlJKzfLZRG/9d/hFwVUQ8Sb4kfUZKyVn72iEifgPsAgyIiOnAWUBP6HjuOXKfJEkNpJYu9UuSpBVk8EuS1EAMfkmSGojBL0lSAzH4JUlqIAa/JEkNxOCXJKmB1MwAPpKKFxG7kAcIaW12sO7Ac8CLwL7AB61ssypwUkrpqq6pUFJ7GfySlufGlNIhS66MiE3IczZAHsL51la2+T9dXZyk9vFSvyRJDcTglySpgRj8kiQ1EINfkqQGYvBLktRADH5JkhqIwS9JUgMx+CVJaiAGvyRJDcTglySpgRj8kiQ1EMfql7QsHwA7RsT0pXx+P3kCn7ERsbR9nNoVhUnqmEgplV2DJEkqiJf6JUlqIAa/JEkNxOCXJKmBGPySJDUQg1+SpAby/wBIg3ZI06SKXQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_va_recall(precisions, recalls):\n",
    "    plt.plot(recalls, precisions, 'b--', linewidth=2)\n",
    "    plt.xlabel(\"召回\", fontsize=16)\n",
    "    plt.ylabel(\"精度\", fontsize=16)\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    \n",
    "plt.figure(figsize=(8,6))\n",
    "plot_precision_va_recall(precisions, recalls)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC 曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 本质  真正类率tpr和假正类fpr （错误的分为正类的负类实例比例）\n",
    "# 与召回/精度曲线相似\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, thresholds = roc_curve(y_train_8, y_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 20551 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 27491 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 31867 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 29575 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 20551 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 27491 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 31867 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 29575 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:211: RuntimeWarning: Glyph 30495 missing from current font.\n",
      "  font.set_text(s, 0.0, flags=flags)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\matplotlib\\backends\\backend_agg.py:180: RuntimeWarning: Glyph 30495 missing from current font.\n",
      "  font.set_text(s, 0, flags=flags)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAAF8CAYAAAAuF9n2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd3hUVeLG8e/JpBcgQAClIyBFCWCApUmxINhQLKHjilhAXZEVF1cXFf3JWtBVFFSwd9eVVREQQVABFTQgSC9KDRAggfTMnN8fCVlASoLJ3Mmd9/M8eTYzc3PzZhbz5px777nGWouIiIgEhxCnA4iIiIj/qPhFRESCiIpfREQkiKj4RUREgoiKX0REJIio+EVERIKIX4vfGDPdGLPbGLPyBK8bY8y/jDEbjDErjDFt/ZlPRETE7fw94n8VuOQkr/cGmhR9jABe8EMmERGRoOHX4rfWLgT2nWSTK4HXbaElQBVjzBn+SSciIuJ+gXaMvzaw9YjH24qeExERkTIQ6nSAY5jjPHfcNYWNMSMoPBxATEzMec2aNSvPXCIicgRrD/9ytlgLXp+lwGfx+ixZeQUU+AqfN4bi172+w7/ObfEv9iNXjc/O9xLuCfnfL30LXmvxBdjS8oeL6nCqME8IBT4fHmMIDw3BALkFPiLDPISY49XaEfs6+ctHfU9vQQEZe7aTn5NFZGxlcg6l77XWJpQ2f6AV/zag7hGP6wA7jrehtfZF4EWApKQku3Tp0vJPJyJSgfh8lk17D5GZ6yU1I4d9mXlk5nmx1rJ650FqVIrAZ/9X3D5rSc/KZ9lv+4tLzGct61IPER8dhs8W7vNgbsFJv6+n6KO0qpRkm+gwwj0hhBiDJ8RgDGzbn03belXYeyiPTmdVo2nNOPK8PhonxBIbGYoBQkIMhsNFawgxYMz/ngsPDaFSZBihIYaYiFDCPCHF2xz+Xydt27aN1q1bE+4xvPTqqwwZMoSQkJBfT2dfgVb8/wVGGWPeBToA6dbanQ5nEhHxG2stG/dkkpVXwKHcAg5k5bN5byZb92VRJTqcA1l5bN2fxY4DOSTERWCtZceBHNIyc/EYQ2aelzCPKR5xl5X9Wfm/ey7EQFSYh5AQw6HcAjymsIjb1osnPTuftvXjaVAtmspRYYQYQ3x0+FFFHHJECYcUfW1oSAhxkaEYA54QQ4gp/KgWE05IiLPl6wRrLcYYateuzciRIxkwYABnn332H9qnX4vfGPMO0B2obozZBvwDCAOw1k4BZgJ9gA1AFnCDP/OJiJSFXek5bEnLJCM7n32ZeazemUF0RCjp2fnszsgFCqfFl27ZT72q0fyyM4PqseHkey3p2b8v2BPZvDfzuM/ne3/f+G3rVWHb/mzqVY2mUUIM0eGhZOd5aVA9prh4D5dx9dgIYiI81KwUSWhICJ4QQ2RYCDHhoXg8htAQQ2SoJyiL2J/Wrl3LjTfeyEsvvUTz5s158MEHy2S/fi1+a23/U7xugZF+iiMickqZuQXsPZRLRnYB2w9kkZaZx75Deazckc6Kbelk5hYQHxNetF1eqff/y84MgN99bXx0GPWrxbDnYC4Nqkfj80Gd+Cia1IzF64OYCA9xkaHUrhKNp6iAE2IjiAwLoUp0OKEhRsVcQVlree211xg1ahSRkZHs2rWL5s2bl9n+A22qX0SkTOXke8nIzmfPoVw27clkf1bhCLxqTOEIO2XrAepUiWLj3kxS03PIzvdSJTqMX9OySvw9MnKOf8y7akw4repUJiffS3ioh05nVcMAdeKjiQ73FB+jrhEXSUyEh9iIUEI9IUSHaTQdrDIyMrj11lt5++236d69O2+++Sa1a5ftxW0qfhGp8PK9Ppb9up+New4x46cd/LYvi10ZOSX++u+PeXyi6fZmtQpPGqtVKZKmNeOICAvhrIRY2taLJyI0hFCPoVpMBGEe4/jJYFIxPfXUU7z33ns8/PDD/O1vf8PjOZ3TJE9OxS8iAelAVh4Hcwrw+ix7D+WydX8Wv6Vlsz8rj817M0nZeoDsfC95Bb4S77NRQgy70nPo0awGEZ4QmtSMwxgo8PqoWzUar89Sq1IktSpHEuYJITrcQ9WYcJW4lCufz8euXbs488wzuffee+nTpw/t27cvt++n4hcRvyrw+lixPZ307Hx2HMgGYOmW/aRm5LBoYxo14iLYfTD3tPYd7gkhqUE81WIjuPa8OrSqU5lKkWGaNpeAlZqayrBhw1i3bh3Lly8nNja2XEsfVPwiUob2ZebxzYa9/LztAMu3ppOd7yUzr4DNezOJjw5nX+apT347svTDPSGcUSWS0BDDvsw8+px7BmdWiaJSVBj5BT5qx0fRqHoMNSpFUjkqrDx/NJEy98UXXzB48GAOHDjApEmTiImJ8cv3VfGLSKkcXvwlM6+A5+dvYP3uQ2zbn33Krzuy9MNDQ8gr8HFh85ps259FUoN49mfl07FRNbo0rk5cZCjR4aFEhZf98U0RpxUUFHD//fczceJEmjVrxhdffMG5557rt++v4heRYgVeH/uy8kjPymd/Vj5b92WR5/Xxy44M3vn+N6LDPSc8g/1ILc+sRMPqMSTWqULdqlHUqhxFTNHx8vjo4FyIReQwYwyLFi1i+PDhPP3000RHR/v1+6v4RYKI12f5eXs6s1buYsveTLYfyOa3fVkkxEWwYfehU379kaXfuEYstSpFsm1/Fvf2bk6belWoHBVGZJhG6SLH8+GHH9K1a1dq1qzJ7NmziYyMdCSHil/EpdKz8vn05x3MSNlBym8HyPOe+Oz3412+1qxWHNv2Z3NG5UiSGlTF6/Nxfbu6tDyzcuFa6Rq1i5RIZmYmd955J9OmTWPMmDE8/vjjjpU+qPhFXGHL3kze+f43pi7cRFxkKLn5vpMWfe0qUUSHe7iwRU3qV42mQfUYasRFEBcZRkJchB+Ti7jbihUruP7661m7di3jxo1j/PjxTkdS8YtUFLkFXpZvTWfemt2s2pFO5agwPl3x+3tYHTzmGPxZCTG0qRdP1ybV6dGsBnERobouXcQPPv/8c6666iri4+P54osvuOCCC5yOBKj4RQLK7owcvl6/l72HcpnzSypxkaGs2pHBnhJe1373RU3p3KQ6TWrEEhHqITw0pJwTi8iJtG/fnv79+zNx4kRq1KjhdJxiKn4RB2zZm8maXQd574ffqBwVxscpO0r8tU1qxOIJMVyeeCZnJcQQEeahS+PqhHlU8iJO+/rrr3nmmWd45513qFatGq+88orTkX5HxS9Sjrw+S8rW/fznp+2s2XmQpb/uL9HXnZUQwxWJtUmIi6Bh9RgaVI+mZlykTqgTCVBer5dHHnmEBx98kEaNGrF9+3YaNGjgdKzjUvGLlIG8Ah9rdx1kyaY0fvxtP1vSsvD5LGtTD57069rWq4LPwk1dG1ElOozz6sfrcjiRCmbbtm0MHDiQhQsXMmjQIJ5//nni4uKcjnVCKn6R0/TztnRueXMZ2w+cetU6T4ihx9kJ9GxWkwtb1CAhNkIn2Im4RHJyMikpKbz22msMGTLE6TinpOIXOQFrLZv2ZrJw3R6+3ZDGzvRsjIHdGbnHvYlMmMdQNSacZrUqUbdqFOfWrkznxtWpE+/fVblEpPzl5OTg8/mIjo5m6tSphIWF0bRpU6djlYiKX4Kez2dZtSODLWmZ+KxlRsoOfvptP/uzjn9P9iMZA1e1qc3Efq10cp1IkFi7di3XX389SUlJvPzyy7Rs2dLpSKWi4pegZK1lza6D3PPhCn7enn7SbWPCPcRGhnJh85rFC97ER4cTExGqy+VEgoi1lldffZVRo0YRFRXFhAkTnI50WlT8EjS2H8jm/R+28syX6wkNMRT47FGv164SBcCfGlUD4Pp2dWldt4rKXUTIyMjglltu4Z133qF79+68+eab1K5d2+lYp0XFL661OyOHf81bz7vfb/1dyR/5+LGrz+W6pLq6VE5ETmjv3r3Mnj2bCRMmcO+99+LxVNyrb1T84hopWw9w13spbN6bSYiBY7q+WMdG1RjWuQG9Wtbyb0ARqVB8Ph//+c9/uPrqq2nUqBEbN26kSpUqTsf6w1T8UqFZa7n9nZ/47Oed2COK/sjS79ioGte3q8sl59TSNfIiUiKpqakMHTqU2bNn8+mnn3LppZe6ovRBxS8V0P7MPJ6dt4HVOzNYvCntd69f2Lwm469oQc1KkTrTXkRK7YsvvmDw4MGkp6czZcoU+vTp43SkMqXilwrB57N8vnIXT8xZy+a9mUe95gkx1KsazYe3dKRarG4pKyKnb+LEidx77720aNGCuXPncs455zgdqcyp+CVgzV61iw+XbWPpln3Hvaa+z7m1GNmjMWfXjCNUI3sRKQNt2rRhxIgRTJo0iehody6+peKXgJKVV8D9H6/iq7W7ScvM+93rl557Bg9cXjiNLyJSFt577z22bt3KmDFjuPjii7n44oudjlSuVPziqAKvj/eXbuPjlO18v3nf714PMXBjl4acVz+eHs1qEBGqk/NEpGxkZmZy5513Mm3aNLp27cpf/vIXQkPdX4vu/wkl4BzMyefx2WtZs+vgccseoF7VaN6/uSO1KmtkLyJlb8WKFVx//fWsXbuWcePGMX78+KAofVDxi5/szshhyoJNTP928+9eiwwLoVpMBH/u0pDe59TijMqRunOdiJSbffv20blzZ+Li4pg7dy49e/Z0OpJfqfilXC3emMZDn/7C6p0Zv3uta5PqPHzlOdSrGq1V80Sk3GVnZxMVFUXVqlV57bXX6Nq1KwkJCU7H8jsVv5Q5ay3PztvA1AUbyczzHvVah4ZVGd61ERc2r6FRvYj4zddff83AgQN57rnnuOKKK7j66qudjuQYFb+UmfTsfJ6YvZY3lvz6u9eeSW7NFYlnquxFxK+8Xi8TJkzgoYceolGjRhX2xjplScUvp81ay8L1e5m6YCO7MnLYtOfohXVa163CnRc2ocfZNRxKKCLBbNu2bQwcOJCFCxcyePBgJk+eTFxcnNOxHKfil1LbmZ7NP2asYs4vqcd9vWuT6vzf1edSJ96di1+ISMUwd+5cli1bxmuvvcaQIUOcjhMwVPxSItZanpyzjufmb/jda81qxXFD5wZc0Lwm1bVkrog4KCcnh59++omOHTsydOhQLr74Ys4880ynYwUUFb+c0qod6Vz6r29+9/zfL23OtUl1qRwV5kAqEZGjrVmzhuTkZDZs2MCWLVuoXr26Sv84VPxyQrsP5tD+kS+Pei4yLIRpQ9vRuXF1h1KJiBzNWsurr77KqFGjiI6O5v3336d6df2OOhEVv/yOtZYJn61m2jdHL7Yz+y/nc3YtnRgjIoHD6/UyZMgQ3n77bXr06MGbb76pUf4pqPil2PrUgzz1xTo+X7nrqOcfvKIlQzs1cCaUiMhJeDweEhISmDBhAvfeey8ej+7ncSoqfmF/Zh63vrWMJZuOXje/z7m1ePLa1kSF6z8kEQkcPp+PJ598ku7du9OuXTuefvpppyNVKCr+IPfiwo08OnPNUc/d0bMxQzs1oJrO0BeRAJOamsqQIUOYM2cOd911F+3atXM6UoWj4g9Sv6VlceFTC8jz+oqfu+eSs7m121laXU9EAtKcOXMYPHgwGRkZTJ06lZtuusnpSBWSij/I5BZ4uXbKYlZsSy9+rlmtOGaM6qx73YtIwPriiy/o1asXLVu2ZN68ebRs2dLpSBWWij8IWGuZkbKDCZ+tZu+h3KNeu6XbWdzbu5lDyURETs7r9eLxeOjRowdPPPEEt912G1FRUU7HqtBU/C73+uItPDBj1e+eT6xTmY9u64xHt8MVkQD17rvv8o9//IOFCxdSs2ZN7r77bqcjuYKK36W8PkvXifPYkZ5z1PP/vKYV1yXVdSiViMipZWZmcscddzB9+nQ6depEfn6+05FcRcXvQl+uTuXG15YWP65bNYr3RnTkzCqaHhORwJaSkkJycjLr1q3jvvvuY/z48YSGqqrKkt5NF0k7lEu3x7/iUG5B8XPj+jRjxPlnOZhKRKTkHn30UTIyMpg7dy49e/Z0Oo4rqfhd4qMftzH6/eXFj6vHRjDp+kS6NklwMJWIyKmlpaWRlZVF3bp1mTJlCl6vl4QE/e4qLyr+Cm7b/iy6TJx/1HOPXX0uye3rOZRIRKTkFixYwMCBA2nUqBELFiygatWqTkdyvRCnA8jpm5Gy/ajSj4sI5ZeHeqn0RSTgFRQUMH78eHr27ElUVBSTJk3S4mF+ohF/BeTzWe77eCXvfP9b8XMf3NKRdg30l7KIBL7U1FSuu+46Fi5cyODBg5k8eTJxcbrzp7+o+CuYLXsz6f7EV8WPG1aP4ePbOlM5Osy5UCIipRAbG0tWVhavv/46gwcPdjpO0FHxVyCHcguOKv2r29TmqetbOxdIRKSEcnJyePzxxxk9ejQxMTF89913hIToaLMTVPwVxCfLd3D7Oz8VP54+LImezWo6mEhEpGRWr15NcnIyK1asoFmzZlx77bUqfQf5/Z03xlxijFlrjNlgjLn3OK9XNsZ8YoxZboxZZYy5wd8ZA03ShLlHlf5LQ1T6IhL4rLVMmzaNpKQkdu7cyWeffca1117rdKyg59fiN8Z4gMlAb6AF0N8Y0+KYzUYCv1hrE4HuwJPGmHB/5gwUGTn59H7m66NurDN3dDcuaqHSF5HA9/DDDzN8+HA6duzI8uXL6dOnj9ORBP9P9bcHNlhrNwEYY94FrgR+OWIbC8SZwus6YoF9QMGxO3K77zfvY+j078nO9wKFy+4uGNODEN1UR0QCnLUWYwyDBg0iMjKSu+++G49Ht/0OFP4u/trA1iMebwM6HLPNc8B/gR1AHHC9tdbnn3jOs9ZywVML2LQnE4Awj+HZ/m255JxaDicTETk5n8/HE088wbJly3j33Xdp1KgR99xzj9Ox5Bj+PsZ/vOGqPeZxLyAFOBNoDTxnjKn0ux0ZM8IYs9QYs3TPnj1ln9QhN72+rLj0Ab64q5tKX0QC3q5du7jkkksYO3YsPp+P3NzcU3+ROMLfxb8NOPKesHUoHNkf6QbgI1toA7AZaHbsjqy1L1prk6y1SW5Z03ny/A3MXZ0KwLm1K7Pp0T40qB7jcCoRkZObPXs2iYmJfP3110ydOpX333+fyMhIp2PJCfh7qv8HoIkxpiGwHUgGBhyzzW/ABcDXxpiawNnAJr+mdMDfPlrBO98XHgW5o2djRl98tsOJREROLSsri6FDh5KQkMC8efNo2bKl05HkFPxa/NbaAmPMKGA24AGmW2tXGWNuKXp9CvAw8Kox5mcKDw2Mtdbu9WdOf7LW0vmxeexIzwGgTnyUSl9EAt5vv/1G7dq1iY6OZs6cOTRp0oSoqCinY0kJ+P06fmvtTGttU2vtWdbaR4qem1JU+lhrd1hrL7bWnmutPcda+6a/M/rL7oM5tHtkbnHpX9yiJgv/2sPhVCIiJ/fOO+9wzjnn8M9//hOAVq1aqfQrEK3c56D2j3xZ/PnUwefRq6VO4hORwJWZmcntt9/OK6+8QqdOnRgw4NgjtVIRaM1Eh/Sd/G3x588kt1bpi0hAW7FiBeeddx6vvvoq9913HwsWLKB+/fpOx5LToBG/A174aiMpWw8A0L5hVa5sXdvhRCIiJ5ednU12djZz586lZ8+eTseRP0Ajfj/7blMaE2etAeDmbo14/+aODicSETm+tLQ0pk+fDkCHDh1Yv369St8FVPx+NPeXVK5/cQkAzWrF8bfezR1OJCJyfAsWLCAxMZFbb72VX3/9FYDw8KC8bYrrqPj9ZPx/VzH89aXFj/99aycH04iIHF9BQQH/+Mc/6NmzJ9HR0SxevFjH8l1Gx/j9YPL8Dby6aAsAiXWr8OaN7YmJ0FsvIoHFWsvll1/OrFmzGDp0KM8++yxxcXFOx5IypvYpZ+8v3crjs9cCcHbNOGaM7OxwIhGR4zPGMGDAAAYNGsTAgQOdjiPlRMVfTnw+y5gPlvPRT9sBaFAtms/v7OpwKhGRo+Xk5DBmzBiSkpIYNmwYgwcPdjqSlDMd4y8H1lp6Pb2wuPQvaFaD+WO6ExJyvJsTiog4Y/Xq1XTo0IHJkyezceNGp+OIn2jEXw4e+3wN63cfAuDSVmfwXP82GKPSF5HAYK1l+vTp3HHHHcTExDBz5kx69+7tdCzxE434y9j/zVzN1IWFNxM8r348kwe0VemLSED54YcfGD58OB07dmT58uUq/SCjEX8Z2rw3k5e/2QwU3mXvw1u0OI+IBI49e/aQkJBA+/btmTVrFhdeeCEej8fpWOJnGvGXkczcAno88RVen6V13Sp8M7anRvoiEhB8Ph+PPfYYDRo0YOnSwvVEevXqpdIPUhrxl4G8Ah8XT1pY/PjZ/m0cTCMi8j+7du1i8ODBzJ07l2uuuYbGjRs7HUkcpuIvA20emkNmnheAKYPOo27VaIcTiYjA7NmzGTJkCBkZGUydOpWbbrpJM5Gi4v+jxn64orj07+3djEvO0e11RSQwLFq0iBo1ajBv3jxatmzpdBwJEDrG/wesTz3Ie0u3AtClcXVu6XaWw4lEJNht3LiRRYsWAXD//ffz/fffq/TlKCr+03QgK4++k78tfjxtWJKDaURE4O2336ZNmzYMHz4cn89HaGgoUVFRTseSAKPiPw25BV7O/+f84in+t4Z3ICJUZ8eKiDMOHTrEDTfcwMCBA2nVqhWzZs0iJES/3uX4dIz/NNz775/JyCkAYNL1iXRuXN3hRCISrHbv3k3Xrl1Zv349999/Pw888AChofrVLiemfx2ltGH3If5TtAb/zec34qo2dRxOJCLBLCEhgR49ejB16lS6d+/udBypADQXVAqZuQVc+NQCABLiIrjnkmYOJxKRYJSWlsagQYPYvHkzxhimTJmi0pcSU/GXwj0frij+/JVh7fDobnsi4mcLFiwgMTGRDz74oHgVPpHSUPGXUG6Bl4Xr9gDQvkFVzqld2eFEIhJMCgoK+Mc//kHPnj2Jjo5myZIlXHvttU7HkgpIxV9C7/+wlYO5BYR5DK/f2N7pOCISZJ544gkeeughBg8ezI8//kibNloaXE6PTu4rgfSsfO6fsQqA0RedTWSYLt0TEf84dOgQsbGxjBo1iiZNmtCvXz+nI0kFpxF/CUz47BcAIsNCGN61ocNpRCQYZGdnM3LkSDp06EBWVhaxsbEqfSkTKv5T2H4gmw+WbQPg/staEObRWyYi5euXX36hQ4cOPP/881xyySW6Ll/KlP41ncLri7YAEB8dxsAO9Z0NIyKuZq1l2rRp3HHHHcTGxjJz5kx69+7tdCxxGRX/SeTke5m6cBMAj1+T6HAaEXG7goICXnzxRTp16sQbb7zBGWec4XQkcSEV/0m88u0WoHCxngua13A2jIi41nfffUeTJk2oWrUqM2fOpGrVqlprX8qN/mWdQE6+l4mz1gDQoWFVjNFiPSJStnw+H4899hidO3fm73//OwDVq1dX6Uu50oj/BO56L6X480f6nutgEhFxo507dzJ48GC+/PJLrrvuOh599FGnI0mQUPEfh9dn+XzlLgDGXtKMytFhDicSETdZsmQJV1xxBYcOHeKll17ixhtv1Kyi+I2K/zimfbOp+PMhHXUmv4iUrQYNGtCqVSv+9a9/0aJFC6fjSJDRgaTjeHRm4bH9K1ufSUyE/jYSkT9u48aN3HHHHXi9XmrVqsXcuXNV+uIIFf8xZhVN8QNM7NfKwSQi4hZvvfUWbdq04Y033mDdunVOx5Egp+I/xuOzC0f77RtW1Zr8IvKHHDp0iGHDhjFo0CASExNZvnw5zZs3dzqWBDkV/xGW/bqPjXsyAXhCC/aIyB907bXX8vrrr/PAAw8wf/586tWr53QkEZ3cd6SJn68FILFuFepVi3Y4jYhURNZaCgoKCAsL48EHH2Ts2LF0797d6VgixVT8Rbbuy+L7LfsAuLVbI4fTiEhFtHfvXm644QbOOussnn76adq3b+90JJHf0VR/kWnfbC7+vFfLWg4mEZGK6KuvviIxMZE5c+bQqJEGDxK4VPyAz2f5ZPkOAF7/c3stpCEiJVZQUMADDzxAz549iY2NZcmSJdxxxx1OxxI5IRU/8MXqVNIy86hZKYKuTao7HUdEKpAtW7bwxBNPMHToUJYtW0abNm2cjiRyUjrGD0xZsBGALo0TNNoXkRL5/vvvadeuHY0bN2blypWa3pcKI+hH/HkFPlK2HgDgitZnOpxGRAJddnY2t956Kx06dODjjz8GUOlLhRL0I/45v+zC2sLPuzVNcDaMiAS0VatWkZyczMqVKxkzZgyXXnqp05FESi3oi//T5TsB6H62Sl9ETuyNN97g5ptvJjY2ls8//5xLLrnE6UgipyWop/pz8r3MWlW4Nv/wLpqqE5ETi4uLo0uXLqxYsUKlLxVaUBf/M1+uB6ByVBhddDa/iBxj8eLFTJ8+HYC+ffsye/ZsatXSOh9SsQV18X/xSyqgaX4ROZrP5+P//u//6Nq1KxMnTiQvLw9AV/2IKwRt8a/ddZANuw8BMObisx1OIyKBYufOnVx88cWMGzeOfv368d133xEeHu50LJEyE7Qn9x1eqe/8pgnUraob8ogIHDx4kLZt25Kens7LL7/Mn//8Z43yxXWCtvjnr90NQKPqMQ4nERGn+Xw+QkJCiIuLY/z48XTt2pUWLVo4HUukXATlVP836/eyakcGAH+5sInDaUTESRs2bOBPf/oTc+fOBeDmm29W6YurBWXxPzFnLQDtG1alSrSO3YkEqzfffJM2bdqwYcMGcnNznY4j4hd+L35jzCXGmLXGmA3GmHtPsE13Y0yKMWaVMWZBWWdYsa1wid6LW9Qs612LSAVw6NAhhg0bxuDBg2ndujUpKSlahU+Chl+L3xjjASYDvYEWQH9jTItjtqkCPA9cYa1tCVxblhkO5uTjK1qi98LmKn6RYPT+++/zxhtv8MADDzB//nzq1avndCQRv/H3yX3tgQ3W2k0Axph3gSuBX47YZgDwkbX2NwBr7e6yDPDqt1uKP2+gE/tEgoa1lvXr19O0aVNuuOEGzjvvPBITE52OJeJ3/p7qrw1sPeLxtqLnjtQUiDfGfGWMWWaMGXK8HRljRhhjlhpjlu7Zs1dLchQAACAASURBVKfEAX74dT8Awzo1KE1uEanA9uzZw+WXX0779u3ZuXMnxhiVvgQtf4/4j3dBrD3mcShwHnABEAUsNsYssdauO+qLrH0ReBEgKSnp2H2c0MJ1hX8kXJdUt+SpRaTCmj9/PgMHDiQtLY0nnnhCS+5K0PP3iH8bcGTj1gF2HGebWdbaTGvtXmAhUCZ/mu/OyCn+/OxacWWxSxEJUNZa7r//fi644AIqVarEd999x+23364FeSTo+bv4fwCaGGMaGmPCgWTgv8dsMwPoaowJNcZEAx2A1WXxzT9dsbP4c0+I/uMXcTNjDNu2bWPYsGEsW7aM1q1bOx1JJCD4darfWltgjBkFzAY8wHRr7SpjzC1Fr0+x1q42xswCVgA+4GVr7cqy+P7fbU4D4JKWmuoTcauPPvqIJk2acO655/LSSy8RGhq0C5SKHJff/4uw1s4EZh7z3JRjHj8OPF7W3/tAVj4AXZvqFrwibpOdnc3o0aOZMmUKgwYN4o033lDpixxH0PxXkVfg47vN+wC4SNfvi7jKqlWrSE5OZuXKlfz1r39lwoQJTkcSCVhBU/yzVu0CCm/KU6NSpMNpRKSsLFmyhJ49exIXF8esWbPo1auX05FEAlrQrNX/xS+pANSqrNIXcZO2bdty8803s3z5cpW+SAkETfH/WLRwT7sGVR1OIiJ/1OLFi+nWrRv79+8nPDycSZMm6fp8kRIKmuI/rEMjFb9IReX1enn00Ufp2rUrv/32Gzt2HLsMiIicSlAc488t8LL9QDYAberGO5xGRE7Hzp07GTx4MF9++SXXX389U6dOpXLlyk7HEqlwgqL4l2wqPJvfGIgK9zicRkROx1133cWiRYt4+eWX+fOf/6wV+EROU1BM9f972TYAhnZs4GwQESmVvLw89u7dC8CkSZNYtmwZN954o0pf5A8IiuL/7/LC44BNa2p9fpGKYv369XTq1InrrrsOay1nnHEGzZs3dzqWSIUXFMUfGVb4Y+rEPpGK4Y033qBt27Zs2rSJO+64QyN8kTLk+uLftj+LnHwfAA2rxTicRkRO5tChQwwZMoQhQ4bQpk0bli9fTt++fZ2OJeIqJTq5zxgz/TT2vc9aO+Y0vq5MzUgpnOZvlBBDiO7IJxLQCgoKWLRoEePHj+e+++7TWvsi5aCk/1VdRuEtdEvanAaYDjhe/L+lZQHQtbFuzCMSiKy1vPHGG1x//fVUqVKFn3/+maioKKdjibhWSYs/11o7rzQ7NgFyUO7wrXi7NklwOImIHGvPnj3ccMMNfPbZZ+Tl5TF8+HCVvkg5K2nx29PY9+l8TZk7fHy/pm7MIxJQ5s+fz8CBA0lLS+Nf//oXN954o9ORRIKCq0/u8/ksuzJyAGiYoBP7RALFlClTuOCCC6hUqRLfffcdt99+u87cF/ETVxf/4WV6wzyG2AidJCQSKM4//3xGjBjBsmXLaN26tdNxRIKKq4t/7a6DAOR7A+Kog0hQ+/e//82oUaMAaNGiBVOmTCEmRjNxIv5W0mFwZWPMo6XYb0DM2f2yMwOAFmdUcjiJSPDKzs7mrrvuYurUqbRr145Dhw4RGxvrdCyRoFXS4r/jNPY9+jS+pkxt3psJQK3KOrFPxAmrVq0iOTmZlStXcs899/Dwww8THh7udCyRoFai4rfWvlbeQcrDutTCqf7Wdas4nEQk+OTm5tKrVy/y8/OZNWsWvXr1cjqSiFDylfvygF2l2K8BfNba+qeVqoykZuQC0FxT/SJ+k5GRQWxsLBEREbz99ts0bdqUWrVqOR1LRIqU9OS+VGttvVJ81C3FvsuFtZYDWXkAtK2nEb+IPyxatIhzzz2Xp59+Gig8e1+lLxJYSlrOFW4Bn/TsfAp8hRGqxUY4GUXE9bxeL4888gjnn38+oaGhdOnSxelIInICrr24ffXOwuP7muYXKV87duxg0KBBzJ8/n/79+/PCCy9QuXJlp2OJyAm4tviXbzsAQKva+gUkUp7Wr1/P0qVLmT59OsOGDdMKfCIBzrXF/8UvqQCcVUMLhIiUtdzcXL788kv69OlDt27d+PXXX4mPj3c6loiUgGtX7svO8wIQ7nHtjyjiiPXr19OpUycuv/xy1q1bB6DSF6lASjrir22M2UHJVuSzRdsVnHaqMnB41b5z6+iMfpGy8sYbb3DbbbcRHh7ORx99RNOmTZ2OJCKlVNIFfDzlHaS81Ksa7XQEEVe46aabePnllzn//PN58803qVu3rtORROQ0lHQBnyGnse9Ma+2/T+Pr/rDD0/wA1WO1PKhIWUhMTGT8+PH8/e9/x+OpsGMBkaBX0qn+Z4FnKN3Nd4YCjhT/lrTM4s91hrHI6bHW8vTTT1OvXj369etXfGc9EanYSlr8GdbaB0qzY2PMsNLHKRt7DxUu1XtObV3DL3I69uzZw7Bhw5g5cyZDhgyhX79+TkcSkTLiypX7Dk/116oU5VQEkQpr3rx5JCYm8uWXX/Lss8/y6quvOh1JRMqQK6/jP5CdD0BkmC7lEymNFStWcOGFF3L22Wfz+eefk5iY6HQkESljrmzGfZmFN+epHBXmcBKRiiEnJweAVq1aMW3aNJYuXarSF3EpVxZ/Zm7hEgIJcbo5j8ipfPjhhzRq1IiVK1cCcMMNNxAToxUvRdyqpFP9kcaYi0uxX0PprgAoUz9s2QdA1RhdyidyIllZWdx11128+OKLtG/fXmUvEiRKWvyfAcml3Pd7pdy+zJiivzlU/CLHt3LlSpKTk1m1ahVjx47l4YcfJixMh8ZEgkFJi7/BaezbseFDWmbh5XxnJcQ6FUEkoL355pvs3buXOXPmcNFFFzkdR0T8qKTFfzbQhZJP3xtg3mklKgPrUg8BEBfpyosWRE7L/v372b59O+eccw4PPfQQo0ePpkaNGk7HEhE/K2kzeq21m0qzY+PQknkFXh+hIYYCn6VajE7uEwH49ttvGTBgAOHh4axevZrw8HCVvkiQct0CPvsy8yjwWapEhxEVrvXEJbh5vV4mTJhAt27dCA0N5a233iI0VDNhIsHMdb8BdmUUXo9cq1Kkw0lEnJWens5VV13F/Pnz6d+/P1OmTKFSJS1jLRLsXFf82/ZnAxDmceUSBSIlFhcXR6VKlZg+fTrDhg3TDatEBHDhAj5b92UBaJpfglJubi733XcfO3bsICQkhP/85z/ccMMNKn0RKVbSEX81Y8zbpdivY79lth8oHPFHq/glyKxbt47+/fvz448/UqdOHW699VYVvoj8TkmLv89p7PuZ0/iaPyy16Bh//arRTnx7EUe8/vrr3HbbbURERPDxxx9z5ZVXOh1JRAJUiYrfWrugvIOUFV/RtQT1qmn5UQkOzz//PCNHjuT888/nrbfeok6dOk5HEpEA5rqT+1K2HgA04hf383q9eDweBgwYQE5ODnfeeScejw5xicjJue7kvoTYwkV7tGqfuJXP5+Opp56iS5cu5ObmUqVKFUaPHq3SF5EScV3x/7IzA4D6muoXF9q9ezeXXXYZd999N7Vq1SInJ8fpSCJSwbiq+PMKfMWfJ8RpuV5xly+//JLExETmzZvH5MmT+eijj6hcubLTsUSkgnHVfHh6dn7x554QXcYk7uH1ern77ruJj49n9uzZtGrVyulIIlJBuar4D9+Ot3EN3Y5X3OHXX3+latWqxMXFMWPGDKpXr05MjA5jicjpc9VU/870wuOd1WLCHU4i8sd98MEHJCYmMmbMGADq16+v0heRP8zvxW+MucQYs9YYs8EYc+9JtmtnjPEaY64p6b53Higs/jrxupRPKq6srCxuvvlmrrvuOpo1a8a9957wPxMRkVLza/EbYzzAZKA30ALob4xpcYLtJgKzS7P/X9MyAagWqxG/VExr1qyhXbt2vPjii4wdO5avv/6ahg0bOh1LRFzE38f42wMbrLWbAIwx7wJXAr8cs93twL+BdqXZeU6+FwDv4eX7RCqYqKgorLXMmTOHiy66yOk4IuJC/p7qrw1sPeLxtqLnihljagNXAVNOtiNjzAhjzFJjzNI9e/YA8GvRnfmq6hi/VCD79+9n4sSJWGupX78+K1euVOmLSLnxd/Ef7xq7Y4fnTwNjrbXek+3IWvuitTbJWpuUkJAA/O86ft2ZTyqKb7/9ltatW/P3v/+dZcuWARAS4qpzbkUkwPj7N8w2oO4Rj+sAO47ZJgl41xizBbgGeN4Y07ckO99RdEveujq5TwKc1+tlwoQJdOvWjdDQUL799luSkpKcjiUiQcDfx/h/AJoYYxoC24FkYMCRG1hri89kMsa8Cnxqrf24JDvfklY41a+T+yTQDR06lLfeeov+/fszZcoUKlWq5HQkEQkSfi1+a22BMWYUhWfre4Dp1tpVxphbil4/6XH9kqpZKbIsdiNS5qy1GGMYPnw4F1xwAcOGDcMYrTIpIv7j95X7rLUzgZnHPHfcwrfWDivpfrPz/ndKgIpfAk1ubi5jx44lNjaWCRMm0L17d7p37+50LBEJQq45i2hfVl7x51qnXwLJunXr6NixI8888wyZmZlYq8tNRcQ5rlmrPyu3AICzErSkqQQGay2vv/46I0eOJDIykv/+979cfvnlTscSkSDnmuI/WFT8sRGu+ZGkgvv1118ZMWIEHTt25K233qJ27dqn/iIRkXLmmpZMzyq8JW9cZJjDSSTY/fbbb9SrV48GDRqwcOFCkpKS8Hi0toSIBAbXHONPzSi8QY9OkBan+Hw+nnzySRo3bszHHxdegdqhQweVvogEFNeM+EOKGr/AqxOnxP92797N0KFDmTVrFn379uX88893OpKIyHG5ZsT/y84MABrXiHU4iQSbL7/8ksTERObPn8/kyZP56KOPqFq1qtOxRESOyzUj/siwwunUQ0Un+Yn4y86dO4mPj2f27Nm0atXK6TgiIiflmhH/utSDADSrFedwEgkGW7ZsYcaMGQAMGjSIn376SaUvIhWCa4o/ITYCgNyiO/SJlJcPPviA1q1bc/PNN5OVVXh/iIiICIdTiYiUjGuKP89bWPh14qMcTiJulZWVxYgRI7juuuto1qwZixcvJjpad4IUkYrFNcf4l287AEBEqC6dkrKXlZVF+/btWbVqFWPHjuXhhx8mLExrRohIxeOa4q8RF8GmPZkU+DTVL2UvOjqa/v370759ey666CKn44iInDbXTPVv3psJaKpfys6+ffu47rrrWLRoEQD33XefSl9EKjzXFP/hG56FeVzzI4mDvvnmG1q3bs3HH3/M6tWrnY4jIlJmXNOSUeGFx/a1Vr/8EV6vl4cffphu3boRHh7OokWLuPHGG52OJSJSZlxT/HlFl/GFh7rmRxIHvPXWWzzwwAP079+fH3/8kaSkJKcjiYiUKdec3LczvfAmPeGa6pfTkJaWRrVq1Rg4cCDVq1end+/eGN3xSURcyHUtqRG/lEZubi533nknzZs3Z+fOnXg8Hvr06aPSFxHXcs2IPzTEUOCzRIfrOn4pmbVr15KcnExKSgp33nmnbqwjIkHBNcVf4LOEmMI/AEROxlrL66+/zsiRI4mMjOSTTz7hsssuczqWiIhfuKL4fUXX8oWHhmiKVkrks88+IykpibfeeovatWs7HUdExG9cUvyF/5uTr1X75MSWLl1KlSpVaNy4Ma+88gqRkZF4PDo0JCLBxRVnwtmiEf8ZlSMdTiKByOfz8eSTT9KpUyfGjBkDQExMjEpfRIKSK0b8RTfmIzJMv8jlaLt372bo0KHMmjWLq666ipdfftnpSCIijnJF8R8e8R9er18EYOXKlVx00UXs37+f559/nltuuUXngIhI0HPFVP/hk/vOqx/vcBIJJGeddRZdunTh+++/59Zbb1Xpi4jgkuIvKDq7T5fyyebNmxkwYAAHDx4kKiqKDz74gFatWjkdS0QkYLii+A/fme9QboGzQcRR7733Hq1bt2bmzJmsXLnS6TgiIgHJFcV/+AY9TWvGOZxEnJCZmclNN91EcnIyLVq0ICUlhY4dOzodS0QkILmi+EOKfoqDOfnOBhFHjBw5kmnTpjFu3DgWLlxIgwYNnI4kIhKwXHFWv88W/gXTuIZG/MHCWkt2djbR0dE8+OCDDB48mAsuuMDpWCIiAc8VxZ9b4CUKiNCd+YLCvn37GD58OHl5eXzyySfUr1+f+vXrOx1LRKRCcEVThhXN9e85lOtwEilv33zzDa1bt+bTTz+lZ8+exWs4iIhIybii+H0U/vI/KyHW4SRSXrxeLw899BDdunUjPDycRYsWMXr0aEJCXPFPWETEb1zxW/PwoC/co+v43erAgQNMmTKF/v378+OPP5KUlOR0JBGRCskVx/hz8r1EUHhbXnGXr776ii5dulCtWjV++uknatas6XQkEZEKzRVNGeYp/DEysrWAj1vk5ORw55130qNHD1544QUAlb6ISBlwxYj/8FR/7fgoZ4NImVi7di3JycmkpKRw5513MmLECKcjiYi4hjuKv+jkvsMjf6m4PvroI4YMGUJkZCSffPIJl112mdORRERcxRVNmZNfuGSvjvFXfA0aNKBLly4sX75cpS8iUg5c0ZSH78rn8+ma7orohx9+4KGHHgKgbdu2zJo1i9q1azucSkTEnVxR/IfFx4Q7HUFKwefz8fjjj9OpUyemTZvG/v37nY4kIuJ6rij+w+P8wyN/CXypqan06dOHe+65hyuuuIKUlBTi4+OdjiUi4nquOLnv8BS/jvFXDAUFBXTt2pWtW7fywgsvcPPNN2OM/mgTEfEHVxR/nrfw5L4QlUdAKygowOPxEBoaypNPPknDhg0555xznI4lIhJUXDFEPnwZn+7OF7g2b95Mly5deOmllwC4/PLLVfoiIg5wRVMevkObij8wvffee7Ru3Zo1a9ZQrVo1p+OIiAQ1VzRl8cl9WsAnoGRmZjJ8+HCSk5Np2bIlKSkp9OvXz+lYIiJBzRVNeXjJ3lDdnS+gLF68mFdeeYVx48axYMECGjRo4HQkEZGg54qT+3xFzR+uEb/jrLX89NNPtG3blgsvvJC1a9fSuHFjp2OJiEgRVzWlit9Z+/bto1+/frRv356ff/4ZQKUvIhJgXDHiPyxEC/g45uuvv2bAgAGkpqbyz3/+k5YtWzodSUREjsM1Q+SoMI/TEYLWI488Qvfu3YmMjGTx4sWMHj2akBDX/NMSEXEV1/x21ol9zhowYAA//vgj5513ntNRRETkJFwz1R+m4/t+NWPGDCIjI+nVqxfjxo3TkrsiIhWE39vSGHOJMWatMWaDMebe47w+0BizouhjkTEmsST79ej4vl/k5ORw++2307dvXyZNmgSg0hcRqUD8WvzGGA8wGegNtAD6G2NaHLPZZqCbtbYV8DDwYkn2vedgbllGleNYs2YNf/rTn3juuef4y1/+wowZM5yOJCIipeTvqf72wAZr7SYAY8y7wJXAL4c3sNYuOmL7JUCdkuy4btWoMowpx1q3bh3nnXce0dHRfPrpp1x66aVORxIRkdPg76n+2sDWIx5vK3ruRG4EPi/Jjj2abi4Xh++D0KRJE+677z6WL1+u0hcRqcD8XfzHa2d7nOcwxvSgsPjHnuD1EcaYpcaYpaBr+MvD999/T5s2bVi/fj3GGMaNG8eZZ57pdCwREfkD/F3824C6RzyuA+w4diNjTCvgZeBKa23a8XZkrX3RWptkrU0CCNGIv8z4fD4ef/xxOnfuzP79+0lPT3c6koiIlBF/F/8PQBNjTENjTDiQDPz3yA2MMfWAj4DB1tp1Jd2xpvrLRmpqKn369OGee+7hyiuvJCUlhaSkJKdjiYhIGfHryX3W2gJjzChgNuABpltrVxljbil6fQrwAFANeL7oMrGCw6P6k1Hvl40nn3ySBQsWMGXKFEaMGKFL9UREXMYcPnmrIos4o4m98G/T+eyOrk5HqZDy8/PZsWMH9evXJzs7m82bN9OixbFXWYqISCAxxiwrycD4WK5Z7k4L+JyezZs307VrVy666CJyc3OJiopS6YuIuJhrluzVlHTpvffee8XT+S+//DIRERFORxIRkXLmmhH/htSDTkeoMLKzsxk+fDjJycm0bNmSlJQUrrnmGqdjiYiIH7im+M+tU9npCBVGWFgYa9euZdy4cSxYsIAGDRo4HUlERPzENVP9EaEepyMENGstL7/8MldddRXVq1dn3rx5hIWFOR1LRET8zDUj/vBQ1/woZS4tLY2rrrqKESNGMGXKFACVvohIkHLNiF8n9R/fwoULGThwIKmpqTz11FP85S9/cTqSiIg4yDXFv21/ttMRAs67777LwIEDadSoEYsXL+a8885zOpKIiDjMNfPjzWpVcjpCwLngggsYNWoUP/74o0pfREQAFxV/mEdz/QAzZszgyiuvpKCggISEBJ555hni4uKcjiUiIgHCNcUf7Cv35eTkcPvtt9O3b1+2bt1KWtpxb2ooIiJBzjXFHxrExb9mzRr+9Kc/8dxzz3HXXXexePFiatas6XQsEREJQK45uc8T4pq/YUrFWkv//v3Zvn07n332GX369HE6koiIBDDXFP/O9OA6qz8jI4OwsDCioqJ48803iY+P58wzz3Q6loiIBDjXDJPrxEc5HcFvvv/+e1q3bs2YMWMAaNmypUpfRERKxDXFHwxL9vp8Pv75z3/SuXNnvF4vAwcOdDqSiIhUMK6Z6nf7uX2pqakMGTKEOXPm0K9fP1566SXi4+OdjiUiIhWMa0b8xri7+dPT0/npp5+YOnUqH3zwgUpfREROi2tG/G68jj8vL4/33nuPQYMG0bRpUzZv3kxMTIzTsUREpAJzzYjfbb2/adMmunbtypAhQ/jmm28AVPoiIvKHuab43TTV/84779C6dWvWrVvHhx9+SNeuXZ2OJCIiLuGa4nfLVP+YMWMYMGAA5557LikpKfTr18/pSCIi4iKuOcbvkt6nR48eREZGMn78eEJDXfN/j4iIBAjXNEtIBZ3qt9YyefJkcnNzufvuu7n00ku59NJLnY4lIiIu5Zqp/opY/GlpafTt25fbb7+dr7/+Gmut05FERMTlXFP8uw/mOh2hVBYsWEBiYiKff/45kyZN4j//+Y+rTlAUEZHA5Jqp/rpVK85a/Tt27ODiiy+mfv36LFmyhLZt2zodSUREgoRrir8iTPUfPHiQuLg4zjzzTD788EO6d+9OXFyc07FERCSIuGaqP9Brf8aMGTRs2JDPP/8cgMsvv1ylLyIifuea4g/UEX9OTg6jRo2ib9++NGjQgMaNGzsdSUREgphrij8Qe3/16tV06NCByZMnM3r0aBYtWkSTJk2cjiUiIkFMx/jL0cKFC9mxYwefffYZffr0cTqOiIiIRvxlLT09nYULFwIwYsQI1qxZo9IXEZGA4ZriD4QR/3fffUebNm3o27cvBw8exBhDtWrVnI4lIiJSzDXF72Tv+3w+Jk6cSJcuXfD5fHz22Wc6Y19ERAKSjvH/QXl5eVx22WV88cUXXHvttbz44otUqVLFkSwiIiKn4prid2rEHx4eTsuWLbn22msZPny4lt0VEZGA5pri9+eIPy8vj/vvv5/k5GTatGnDpEmT/Pa9RURE/ggXFb9/vs/GjRvp378/P/zwA5UqVaJNmzb++cYiIiJlwDXFn5FdUO7f4+233+aWW27B4/Hw4Ycf0q9fv3L/niIiImXJNWf1x8eEl+v+//3vfzNw4EBatWpFSkqKSl9ERCok1xR/eU315+XlAXDFFVfwwgsv8NVXX1G/fv3y+WYiIiLlzDXFX9bn9llrefbZZ2nevDl79+4lLCyMW265hdBQ1xwdERGRIOSi4i+75k9LS6Nv377ccccdNG/evMz2KyIi4jT3FH8Z7WfBggUkJiYya9Ysnn76aT755BOqV69eRnsXERFxlmvmrcvqOv4nn3yS6OhoFi9eTNu2bctknyIiIoHCNcX/R3p/69atANStW5dXXnmFiIgIYmNjyyiZiIhI4HDRVP/pNf/HH39MYmIiI0aMAKBatWoqfRERcS3XFH9pL+fLzs5m5MiRXHXVVTRq1Ihnn322fIKJiIgEENdM9ZdmwL9lyxauuOIKfv75Z+6++24effRRwsPLdwEgERGRQOCa4i/NVH+1atWIi4tj5syZ9O7duxxTiYiIBJagmepPT09n7NixZGdnExcXxzfffKPSFxGRoOOa4j/ZAj5LliyhdevWPPnkkyxcuPCU24uIiLiVi4r/98/5fD4ee+wxunTpgrWWr7/+ml69evk/nIiISIBwTfEfb6p/9OjR/O1vf+Pqq68mJSWFjh07+j+YiIhIAHHNyX1Hntbv8/kICQnhtttu45xzzuHGG2/U1L6IiAgOjPiNMZcYY9YaYzYYY+49zuvGGPOvotdXGGNKtG6uMYW30P3rX//KgAEDsNbStGlThg8frtIXEREp4tfiN8Z4gMlAb6AF0N8Y0+KYzXoDTYo+RgAvlGTf237dTOfOnXniiSeIj4+noKCgDJOLiIi4g7+n+tsDG6y1mwCMMe8CVwK/HLHNlcDr1loLLDHGVDHGnGGt3XminXqzDzKgT3fCQj18+OGH9OvXrzx/BhERkQrL31P9tYGtRzzeVvRcabc5ijdjD02atyAlJUWlLyIichL+HvEf72C7PY1tMMaMoPBQAEBuyg/frWzQoMEfSycnUx3Y63SIIKD3ufzpPS5/eo/94+zT+SJ/F/82oO4Rj+sAO05jG6y1LwIvAhhjllprk8o2qhxJ77F/6H0uf3qPy5/eY/8wxiw9na/z91T/D0ATY0xDY0w4kAz895ht/gsMKTq7/09A+smO74uIiEjJ+XXEb60tMMaMAmYDHmC6tXaVMeaWotenADOBPsAGIAu4wZ8ZRURE3MzvC/hYa2dSWO5HPjfliM8tMLKUu32xDKLJyek99g+9z+VP73H503vsH6f1PpvCnhUREZFg4Jq1+kVEROTUKlTxl9dyv/I/JXiPBxa9fVDr6AAABD1JREFUtyuMMYuMMYlO5KzITvUeH7FdO2OM1xhzjT/zuUVJ3mdjTHdjTIoxZpUxZoG/M1Z0Jfh9UdkY84kxZnnRe6xztkrJGDPdGLPbGLPyBK+XvvestRXig8KTATcCjYBwYDnQ4pht+gCfU7gWwJ+A75zOXZE+SvgedwLiiz7vrfe47N/jI7abR+H5MNc4nbuifZTw33IVClcNrVf0uIbTuSvSRwnf43HAxKLPE4B9QLjT2SvSB3A+0BZYeYLXS917FWnEX7zcr7U2Dzi83O+Ripf7tdYuAaoYY87wd9AK7JTvsbV2kbV2f9HDJRSusyAlV5J/xwC3A/8GdvsznIuU5H0eAHxkrf0NwFqr97p0SvIeWyDOFN4pLZbC4teNVErBWruQwvftRErdexWp+MtluV85Smnfvxsp/EtTSu6U77ExpjZwFTAFOV0l+bfcFIg3xnxljFlmjBnit3TuUJL3+DmgOYWLsP0M3Gmt9fknXtAode/5/XK+P6DMlvuVEyrx+2eM6UFh8Xcp10TuU5L3+GlgrLXWq1tKn7aSvM+hwHnABUAUsPj/27tj16bCKAzjz8GK4OBiQSep6OCki+CiUEQQ9R9wqFBXB8FJJx10cHbo1KGbIiIqCAU3HSpOokNRREXc1EXoUKgeh5tCqTf23rSJJN/zW0LCzeXkEO7L/ULOFxEvM/N9v4sbEU16fBp4DZwEDgDPIuJFZv7sd3EFaZ17wxT8WzbuV1016l9EHAZmgTOZ+WNAtY2KJj0+CtzrhP44cDYiVjLz0WBKHAlNrxffM3MJWIqI58ARwOBvpkmPLwK3s/ox+kNEfAIOAa8GU2IRWufeMC31O+63/zbscUTsAx4CF7wz6smGPc7M/Zk5kZkTwAPgkqHfWpPrxWPgRESMRcRO4BiwOOA6h1mTHn+hWlEhIvZQbSrzcaBVjr7WuTc0d/zpuN++a9jj68BuYKZzR7qSbsbRWMMea5Oa9DkzFyNiHngD/AZmM7P2L1P6W8Pv8k1gLiLeUi1JX81Md+1rISLuApPAeER8BW4A26H33HNynyRJBRmmpX5JkrRJBr8kSQUx+CVJKojBL0lSQQx+SZIKYvBLklQQg1+SpIIMzQAfSVsnIiapBn/U7fq1DXgHfAbOAcs1x+wCLmfmXET8ArpNCtubmWMRMQ3cAepmtO8AnmbmdIuPIKlHBr9UrieZeX79ixFxkGovBqhGM8/XHHNrzdNvmVm7PXNn0tiqmcy8VnPMKWCqVeWSeuZSvyRJBTH4JUkqiMEvSVJBDH5Jkgpi8EuSVBCDX5Kkghj8kiQVxOCXJKkgBr8kSQUx+CVJKojBL0lSQZzVL5VpGTi+bpb+WgtUG/jMRkS3c1zpPK784zyrloCpiOg2k//+Bu+XtEUiM/93DZIkaUBc6pckqSAGvyRJBTH4JUkqiMEvSVJBDH5JkgryB+yZWRPJSCuFAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#plt.roParams['font.sans_serif'] = ['SimHei']\n",
    "def plot_roc_cure(fpr, tpr, label=None):\n",
    "    plt.plot(fpr, tpr, linewidth=2, label=label)\n",
    "    plt.plot([0,1], [0,1], 'k--')\n",
    "    plt.axis([0,1,0,1])\n",
    "    plt.xlabel('假正类率', fontsize =16)\n",
    "    plt.ylabel('真正类率', fontsize = 16)\n",
    "    \n",
    "plt.figure(figsize=(8,6))\n",
    "plot_roc_cure(fpr, tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9183685416982093"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算曲线下面积AUC\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_train_8, y_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练随机森林分类器，比较SGD分类器的ROC曲线和 ROC AUC分数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   1. 获取训练集中每个实例的分数\n",
    "   2. RabdomForestClasslfier 没有descision_function(), 但是拥有dict_proda()方法，sklearn中分类器都用这两个中一个。\n",
    "   3. dict_proda返回一个矩阵，每行一个实例，每列代表一个类别的概率，比如这个图片70%是8\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "forest_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
    "y_probas_forest = cross_val_predict(forest_clf, X_train, y_train_8, cv=3, method='predict_proba')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1. , 0. ],\n",
       "       [0.9, 0.1],\n",
       "       [1. , 0. ],\n",
       "       ...,\n",
       "       [1. , 0. ],\n",
       "       [1. , 0. ],\n",
       "       [1. , 0. ]])"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_probas_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 绘制ROC曲线，需要决策值而不是概率，直接使用正类的概率作为决策值\n",
    "y_socres_forest = y_probas_forest[:,1]\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_8, y_socres_forest )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0. , 0.1, 0. , ..., 0. , 0. , 0. ])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_socres_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAFlCAYAAADYnoD9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3xUVf7/8ddJD6GEJiUQijTpagRUBAQF1FXEtosddAEXVoH9Ka6ugmV1XRfsLBYU97sqVlwVhLWsYoFdEBuCKIK0SO8QSJnz++MkmTTIJJnkZm7ez8djHjNz5sy9n7ni/eSce+45xlqLiIiIeCfK6wBERERqOiVjERERjykZi4iIeEzJWERExGNKxiIiIh5TMhYREfFYjFc7btSokW3durVXuxcREalyX3zxxQ5rbeOi5Z4l49atW7Ns2TKvdi8iIlLljDHrSypXN7WIiIjHlIxFREQ8pmQsIiLiMSVjERERjykZi4iIeEzJWERExGNKxiIiIh5TMhYREfGYkrGIiIjHSk3GxphnjTHbjDErjvK5McY8aoxZY4z5xhhzUvjDFBER8a9QWsazgaHH+PwcoH3uYzTw94qHJSIiUnOUOje1tXaRMab1MaoMA/5hrbXAEmNMsjGmmbX2lzDFKCIiES4nBzIy3OvatYPlGzfCnj3QrVuw7PvvLTt2QvsOluT6Fmth02bLqlXQpKmlY0fIsZbduy3TH7Ic3w5GXG4JWAgELE8/YzlwEK4daalVy5V/+KFlxXeWM/rBCZ1d2bp1lvnvWlqmwuAhloB1+3r6Gff5yJGWvu0bERtd+Vd0jcuhpVRyyfgda23XEj57B/iLtfbT3PcfAJOttcVWgTDGjMa1nklNTT15/foS58sWESmVtbknXxs8iQYKlgWO8XnAFqqbt62cQMnbstaSEyjbtgLWkhMoeVt5+yr8G8jdT8nbKrovawvvt/DvDX6eEwi+zsyCrGxLdDSYKLevrGzIyLCYKEtcPPnf2bXLYrHUqxf83Tt2QmamJbmBJSrKlW/ZAhmHLY0aB7+/e7dlz16oW9dSu44rO3zEsnsPxMZaatcJHpf9By3GQFx8wd/t9b+uoG+mDqZuQmzYtmeM+cJam1a0PByrNpkSyko8lNbap4CnANLS0qrR4Ra/OtYJu/jJMPj50U6iRz0hV+LJv0wn7KOckI918i96wi54QswpEHexk7+1BAIUif/Yn+cUrRsI/jfKKeG3FPxeMIbqd8KuKlEGDAZjDDHREGUMBkNmJkRhSEoy+XV27YQjRwwpzQ2xsRAdZdi+zbB3DzRtamjYwH1//z7D2rWQXM/QoYP7vg0YvvrGfaffGYbYKEOUMXyz3rB/L/Tta6if7Opu+8mwayO0a2xom2KINrBiryH9Z0P77pDWwRAVBdu2Gt5ZZmjYDC7s77YXZeD55w2HDsFFl7hYowy8/75hwwbDkMHQto2r+8NqWLTI0PkEOHOAq5dxyDBvniE7y3DNNRBtDMbAggWGzMOG886DpFru+8uXw7q1hl6nGNq3c7/9l1/gk0VuvwPPhKjc3/nCC+4YXn0V1IqNrpL/tuFoGT8JfGStfSn3/WpgQGnd1GlpaVZLKHrDWsueQ1ls3pNBeu5jy74jZGYHSj35h+uEnVOwboHPc4olsuAJu6QTct4JO8cWjaFmn7Cjo9wJO8qQe9JzJ6koY4iOyj1hF/k8Kqp43ZI+L/i96IJ1i3weXaRuVFTePt0Ju+i2Cu4/GF/u51GmSDzH/rzk31r48+ioor81WDfve8U+LzFGOJxhWP+zoV49aN3KfZ6ZCe/92xATA8POD+5r0kSXAGfOdIkifbPhulGwYoVh7utwxhmu7rS/wR13GG6/zXDP3W5f//gHXHMN9O0Ln3zi/ntv2gSdO8P+/ZCVBTExcOQInHsufPghzJoFV1/tyv/wB3j+efjb3+Daa933P/wQfvtbGDQInnrKlR0+jEtkSfDWW8F/W/fd5/Y3eTK0auXK3nvPbeOmm6BpU1d24IDbRlISJCZWyT/7iFGZLeO3gPHGmDlAb2Cvrhd7KzM7wNZ9h/OT7ebdGaTvzWDznsP5yfdQZk6h78RGG+JjogudpMJ9wo6OCv6FXdoJO7rQiTO0E3bxBFTg5B9V8gn5WCfsqKOdkI9x8j/WCbtcJ/8i+yp0XKNK3paUz5dfwr59cMopUKsWHDwIL74I69fDZZdB9+6u3qxZMGYMTJvmEhDA1Klw110wfDi88YYr27YN/vBbaNQIbrw2uJ/334Ldu6HOM9CgNjRoD/WT4PA+d0JOinf1GiRDowbQoD7k/Wft2xeOOw769Qtur0ULuPRSiIoi/4/P+HiXOLOyoGdPl4jBxTxtWuHfPXAg/PRT4bKEBPjgg+LH6LbbipedfbZ7FFS7duHrwlK6UpOxMeYlYADQyBizCZgCxAJYa2cC84FzgTXAIWBkZQUrrtW3LyM72Krd65JtsJV7mK37DxdrETZMiiOlfiLtGtemX/vGNE9OoEX9RJonu0fDpDidyCXiZWW5FlmdOu79++/D3LmuFXfLLa7swAHo0MElr02bgt8dMQJWr4ZVq6BTJ5eYly6Fp5927/OS8Y4dbjDS99/D2rXQti0MHuyScV7LEFxCv/JKqFevcIzPPeeSa1KSex8VBa+84hJofHyw3u9+5x4FtW0LW7cW/92zZhUv69279OMl1UdI3dSVQd3UJcvKca3a9NxW7OY9GYW6kzfvzuBgkVZtXHQUzZMT8hNr8+REWuS/duUJVXTdQ6QisrNh505o0iRY9p//uNbpoEHQsqUre+UVeO0112K95BJXNmeOS6jXXusSHriu3H79XGv3f/9zZVlZEBfnkmB2drDVecUVbmTv7Nku6YHr0l2yBG64IZiMt2xxj0aNXKtUpCwqs5taymDf4axCibVg1/HmPRls3XeYQJG/jxokxdE8OYHWDZM47fhGhVq0zZMTaJQUT1SUWrUSOdLTXYvz0CF44AFXtnw5/OpX8Msvros4IcElzOnT4Z133LXLvGS8ciW8+iqccEIwGdet656XLAnu56STXOty0KBgWWysaxEXvZb5wgvF47zmGvcoqGnTwi1gkXBQMg6j7JwA2/YfKaFFe9hdt92Twf4j2YW+ExttaFbPJdXTjm9ESpEWbkpyIolxatVKZMjMdC1WgMsvd88rVsD48a5F+tlnrmz9enjySZd477nHtVRPPDF4H+p778EFF7jXAwdCw4bQvHlwPxdf7BJx1wJDSocMcS3d6AL/uyQlwRNPFI8zJSU8v1ckXNRNXQYHjmQXaNEGu47T97jBUlv2HSanSLM2uVYszeslklLfJdaC3cktkhNpVFutWqn+1q2Dl1+GPn1gwABX9sEH7ppo//6uixjghx+gY0c3uvezzyA52SXYWrXc55mZrmUK8JvfuCQ7bZprBYNLpjFqIoiPqZu6AtL3ZHDnv1bw/qpthcpjogxN6yWQkpxI7zYNgq3Z+omkJCfQrF4iSfE6xFJ9/fQTLFjgbmNp3dqVXX65S67Tp8OECa7srbfgj3+EX/86mIytdddOd+wIbq9ZMzdwKCfHDaQC1x382mvu+mpUgYmM8hJ4QUrEUlPpn/4xBAKWf/53PQ+8+z0BC+POPJ5OTevmdx83rhNPtFq1EgFyctx11yVLXBdvWhoEAnDhha4bOSkpeN9p164u0f74Y/D7ffrARRdBu3bBsr59YfPmwrew1KlT+JptnosvrpSfJeIbSsZHsWbbfia//i1frN/NGe0bcd/wbrRsUMvrsESKWbbMdRmPHOnuQbUWxo2Dr7+G//f/3L2vxsDixW6wVJs2LhlHRblRwpMnU+hWuHHjYOxYqF8/WNa7N7z+euH9JiQUvo4rIuWnZFxEZnaAv3/0E0/8Zw214qOZdmkPLjopRffgiicOHXLXWPOus552mkuqGza4kcU5Oe6WnB9+gC5d3Gjk7GzXQv38c3etF1ziPfVU19WcVuBqVUn3sha9L1ZEKl/lL0URQZZv2M2vHvuEh97/gSFdm/L+pP5cfHILJWKpEuvWua5jcNdb27Z13ccrCqwkHhfnnl95xT1HR8N117kW8d69riw21t2O8+abwRHNAMOGuVmZTtKK4yLVjkZTAwePZPO3f69m9uc/07RuAvde2JVBJzQp/Ysi5ZCR4VqtHToE75vt29cNdBozxt3KU7s23HorPPSQ635+9llXb/ly16Xco4cGO4lEoqONpq7xLeOPVm9j8EOLmP35z1zVpxX/nthPiVjC5t133Xy+334bLOvVC846y03TmGfAADd14yuvuNuA4uLgxhvh55+DiRhcq/bkk5WIRfymxibjXQczmTDnS659bimJcdG8OuZU7h7WlTphXLdS/M3aYLfyxo1uxHBSUnDaRXAJ9/773SIEeUaPds8F5xieNMnNOvXKK8F7blu3Dq6MIyL+VuOSsbWWf321mbOmf8y8b3/hxkHtmXdjX9JaN/A6NKnGjhwJ3jcLLvFGRbmZosBNr9izp5vof9YstyoPuGXsbr8djj8+OGL59793r//85+D2GjQITowhIjVPjers2rb/MLe89g0frd5Oz5bJPHBxdzo2reN1WFKNHDjg7q9t0QIaN3ZJ89pr4R//gH/+041cBjdgCmDhQjcN46mnuikWU1PdjFR5twVdeKF7iIgcS41qGU996zuWrN3JlPM78/oNpykRC7t3w549wfeXXuquy+Zdz83Ohm7d3Ov09GC9KVPc++nTg2WpqW4Uc95MViIioaoxyXjT7kMsWLGFa05rzcjT22jmrBpm/Xq3Ks/SpcGyJ55w3cP33BMsO/NMN19yXuKNjXVTQK5aBTffHKzXtKmb+lFEJBxqTDL+v8XrMcZw9amtvQ5FKtlbb7mJLP7972DZlCluUYO33w6W1avn7tMtODL5llvcXMtTpwbLWrZ0i8uLiFSWGpGMD2Vm89L/NjC0S1NSkhNL/4JElCNHCi9W8P778Pe/u7mY81x1lbsG3KVLsOyyy9ygrLz1dEVEvFIjBnC9vnwz+w5nM6pva69DkTAIBIKr/6xa5Zbre/ZZN4nG6ae7STK+/Ta4Hi64xeW3FV50K382KxERr/m+ZRwIWJ77bB3dW9TjpNT6pX9Bqq2tW6F9e7dGbt5tQm3auOdRo9xygOAWqf/Pf9zEGiIikcD3yXjRj9tZu/0go05vozmmI8yUKTB7dvB9YiJkZcH+/cFl+hIS3D2+1sLVV3sSpohIhfk+GT/32c8cVyeec7tp6Gt1Fwi4eZsBMjPdovcjR8LgwS7Z1q3r5mr+7jt3X2+elBRv4hURCRdfJ+M12w7w8Q/buapPK+JifP1TI1pmphtgFR0Njz7qknJcHNx9t1t16PHH3Xq84Nbm7dzZ23hFRMLN1xlq9ufriIuJ4vLeqV6HIgXs3w933BFcRzcqyk0ZCfDkk7B9u3s9ZIi7N7hDB2/iFBGpKr5NxnsPZfH6F5u5sGdzGtaO9zqcGm/XLne9F9x9vX/7Gzz/PBw65N537OjW8127Fppo0SwRqWF8e2vTnKUbyMjKYeTpbbwOpUY7eBC6dnVLAX79tbvPNzHRLSuYlRXsfjZG00iKSM3ly5Zxdk6A5z//mVPbNuSEZnW9DqdG+eQTt3rRv/7l3teq5a7zgrsXOCfHvb7jDndNOFFzsIiI+DMZL9+wh/S9h7nqVC0GW9msdSsd5VmxwrWA777bJV5jYOJEtxjDww9rog0RkZL4Mhmv33kQgM5qFVeqjRuhbVt46aXgWr8XXOCWDLznHjc6GtzczvXqeReniEh158trxpv3uJtVmyUneByJ/xw86JYVrFcP6tRxKxyNHg379sEf/uDu+c1bflBERELjy5bx5t0ZHFcnnviYaK9D8ZXbb4fatd21X3DTUt5wgxsF/Yc/eBubiEgk82cy3pNBSn2NDKqILVtgxAj461+DZd27u+f33guWXXedRkGLiFSUf5OxlkqskLlzYc4cNyPWpk2ubNAgWLkS5s/3NjYREb/x3TXjQMDyy57DDO3a1OtQIsrGjTBhAjzzDNSv77qf4+MhPR2OO87VadTIPUREJLx8l4y3HzhCZk6AFmoZh2zPHrjlFnjjDbcow+jR7pakUaO8jkxEpGbwXTf1pt1uJLWuGR9bTg58+KF7nZwMY8a4a7/9+gVnxRIRkarhu5Zx3m1NKcm1PI6k+rLWJeADB9xArSZNYMAANypaRESqnu9axpvVMi5RIABHjrjXxrgpKo2B5cu9jUtERPyYjPccol5iLLXjfdfoL7cff3QTdDzzTHCmrMcec13V55zjbWwiIuLHZLxbtzUVZa0bGT1+fPAe4Xr1dG1YRKS68F8y1oQffPklnHYaPPSQe9+hg3v9009w/vnexiYiIsX5Khlba9UyBj79FBYvhptvDnZLX3ONW9RBRESqH18l470ZWRzMzKFFDWsZHzwIJ54IO3a49zfcAGedBTNmQILWyhARqfZ8lYzz7jGuacl4+nT46iv4y1/coKyYGHdtePRoryMTEZFQ+CoZ16R7jA8dCr4ePdqtpjRkSHANYRERiRz+Ssa5LePmPl/H+O23XfL94Qf3vkkT2LwZzj7b27hERKR8fJWM92ZkAVC/VpzHkVSuffvc7Up//rObzAOgbl1vYxIRkfLzVTI+lJlNYmw0UVH+u4F2xQo3dSW4wVl33w1PPw1RvvovKCJSM/nqVH4oM4dacf67aPrii9CtG3zwgXvfpAnccQfE+bsDQESkxvBfMo73XzLOW09440Zv4xARkcrhqwmcD2VmkxTnj5+0c6db2KF5czjzTHj/fRg0yOuoRESkMoTUMjbGDDXGrDbGrDHG3FrC5/WMMW8bY742xnxnjBkZ/lBLdygzh0QfdFN/+y00auTuH7bW3a6kRCwi4l+lJmNjTDTwBHAO0BkYYYzpXKTaOGCltbYHMACYZoyp8iuaB4/4o2XcrJnrmp42Dfbv9zoaERGpbKG0jHsBa6y1a621mcAcYFiROhaoY4wxQG1gF5Ad1khDEMkt49274eOP3etGjeD++901Yt2yJCLif6Ek4xSg4NChTbllBT0OnACkA98CN1lrA2GJsAwOZeaQFIHJ+L//hQYNYMIE1y0NMGoUtGjhbVwiIlI1QknGJd20a4u8HwJ8BTQHegKPG2OKtemMMaONMcuMMcu2b99e5mBL41rGkddN3b49XHCBm1/6wAGvoxERkaoWSjLeBLQs8L4FrgVc0EjgDeusAdYBnYpuyFr7lLU2zVqb1rhx4/LGfFRuNHXktIw3bnQzaDVoAH/9q7s+XKeO11GJiEhVCyUZLwXaG2Pa5A7K+g3wVpE6G4BBAMaYJkBHYG04Ay1NIGDJyMqhVnxktIwXLIAOHeD7713XdMeObr5pERGpeUpNxtbabGA8sBBYBbxirf3OGDPWGDM2t9o9wGnGmG+BD4DJ1todlRV0SQ5n52AtETEDV04OrFwJhw+725dERKRmC6kZaa2dD8wvUjazwOt0YHB4Qyubg0dyACKimzo6GiZNct3S48eD8d9U2iIiUga+mQ4zI9Ml4+o8gOu//4Vffgm+nzIFGjb0Lh4REakefJOMD2a625qra8t43z64/HJITYWpU72ORkREqhPfJONDuS3j6jqAq25dmDjRDdSaMMHraEREpDrxUTJ2LePqNoBr82bYkTuUbeRIWLoUkpO9jUlERKoX3yTjvAFc1SkZL1vmZtGaM8fdT5yUBImJXkclIiLVjW+ScUZWXsu4+nRTL10KcXHwyisQ5ZsjLSIi4VZ9MlcFVadbm6x1tyvdcAOcfjq0bu11RCIiUp35JhkHb23yNhm/8IKbSeuCC1xC7t7d03BERCQC+KbzNH80tYfd1Dk58NxzcOGFcMstnoUhIiIRxjct4+xAgCgD0VHeTWcVHQ2jR7sR1FOmeBaGiIhEGN+0jLNyLDEejZKyNrgO8QUXwDffaNEHEREJnW+ScU4g4Fmr+IIL4PbbITMTEhIgNtaTMEREJEL5JhlnBywx0VWfjA8cgO++g/vvh+XLq3z3IiLiA765ZpwTsMR40DKuXRteegneew/69Kny3YuIiA/4JhlnByzRVXjNODvbjZ6Oj4fevd1DRESkPHzTTZ2TU3Ut40DArb500UWwYEGV7FJERHzMN8nYtYyrJhlv2ADt2sH8+dC4cZXsUkREfMxH3dSBKhvA1bo1LFrk7idOSamSXYqIiI+pZVwGhw5BVlbwvRKxiIiEg2+ScU6OJbYSB3BlZcGpp0LXrvDkk5W2GxERqYF8k4wru2WcmQm//jWsXQv9+lXabkREpAbyTTLOqeRrxklJMH68m9jjhBMqbTciIlID+SYZV2bLeOtW91y3LnTrVim7EBGRGsw3ybiyZuDatg06d4aFC4OLQYiIiISTb5Jxdk7ltIw3bIAjR+Cee8K+aREREcBn9xnXign/z0lLg5tucrNtGe+WShYRER/zTTLOqcRVm+69V4lYREQqj3+6qcN8zTgjwy2LCErEIiJSuXyTjHPCPJr64Yfhttt0rVhERCqfb5KxaxmH7+dceqlbHnHbtrBtUkREpES+umYczpZxu3bw1VfQsWPYNikiIlIi3yTj7EAg7PcZd+oU1s2JiIiUyD/d1GG6z3jmTBg1CtavD0NQIiIiIfBPMg5YYqIr/nOmTYPnnoP588MQlIiISAh8000drukwX34ZFi2Ca6+teEwiIiKh8E0yzs4JhKWb+qST3ENERKSq+KabuqKjqdPT4eDBMAYkIiISIt8kYwtUpGE8aRKkpsLjj4ctJBERkZD4JhkHrMWUc97KrCzXKj5wAE47LcyBiYiIlMI314ythfI2jGNj4e233VKJ8fFhDUtERKRUvmkZW0u5W8aBgHtWIhYRES/4Jxljy7W60rp18LvfhT8eERGRUPknGdvyDeB6/XV4+ml45ZXwxyQiIhIK3yTjgLWYclw1btMGTj0V9u6thKBERERC4J8BXFCubuqLL4aLLgpeNxYREalqvmkZV2QAlzEQHR3mgERERELki2RsrQXKdmvToUMwbhysWFE5MYmIiITKJ8nYPUeVoWX82mswYwaMGBH8voiIiBd8cc04kNcyLkPT+MQT4bbbKPP3REREws0XyTivYVuWnNqtG7Rvr4k+RETEe/5Ixnnd1GW80TghoRKCERERKaOQrhkbY4YaY1YbY9YYY249Sp0BxpivjDHfGWM+Dm+YxxYo40Xfm2+GKVNg+/ZKCkhERKQMSm0ZG2OigSeAs4FNwFJjzFvW2pUF6iQDM4Ch1toNxpjjKivgY8caWr3vvoN334W0NDj//MqNSUREpDShtIx7AWustWuttZnAHGBYkTqXA29YazcAWGu3hTfMYyvraOoXXoDu3WHQoEoMSkREJEShJOMUYGOB95tyywrqANQ3xnxkjPnCGHN1SRsyxow2xiwzxizbHsY+4kAZ7zOuXx+WLYNatcIWgoiISLmFkoxLynFFL9LGACcD5wFDgDuMMR2Kfcnap6y1adbatMaNG5c52KPJC6Ys9xnHxoZt9yIiIhUSSjLeBLQs8L4FkF5CnQXW2oPW2h3AIqBHeEIsXaj3GVsLl10Gf/kLHDxYBYGJiIiEIJRkvBRob4xpY4yJA34DvFWkzr+AM4wxMcaYWkBvYFV4Qz26UAdTf/UVvPoq3HMPJCZWbkwiIiKhKnU0tbU22xgzHlgIRAPPWmu/M8aMzf18prV2lTFmAfANEACesdZW3azPIQ7gatECZs2C9HSI8sVEoCIi4gchTfphrZ0PzC9SNrPI+weBB8MXWuhC7aZu3BhGjaqCgERERMrAF+3D8kyHKSIiUl34IxnntoyPNR3mzp0wYQLMnl1FQYmIiITIF8k4kNs0PlbL+D//gUcegcceq5KQREREQuaPhSLIu2Z89HTcpYu7pSkpqaqiEhERCY0/knFey/gYTeMTTnAPERGR6sYX3dT5yfgYHdVlXNhJRESkyvgjGed2Ux9t/NYPP8DFF8P+/VUYlIiISIh80U0dKKWbetkymDvXzbr1wgtVF5eIiEgofJGMbf6qTSVn43bt3G1NPapstmwREZHQ+SQZ5744Ssu4Vy/3EBERqY58cc04T1mWUBQREakufJGMjzVSevdueP11+OKLqotHRESkLPyRjPMm/Sjhsy++gEsugYkTqzYmERGRUPnimnGeknqp69aFiy5yM3CJiIhUR75Ixsfqpu7Vy3VTi4iIVFc+6aZ2NH5LREQikS+ScZ6S7jP+8Uc4csSDYERERELki2Rsj9FP3aEDJCRAenoVBiQiIlIG/kjGuc9Fu6kPHYK2bd3rZs2qNCQREZGQ+WIA19HUqgU//QSBgK4ni4hI9eWPlnEpyyNG+eJXioiIX/kkTeVO+lGk+Zud7UUsIiIiZeOTZOwU7Yk+7zxo0QI++cSTcERERELii2R8tG7qhATYvBmSk6s2HhERkbLwxQCuo42mfvNN+PTT4IhqERGR6sgXyThP0Uk/jIEzzvAoGBERkRD5uptaREQkEvgjGeePpg6WLVoEv/oVPPSQR0GJiIiEyGfd1EFffw3z5kFKimfhiIiIhMQXybikburzz3cDt1JTqz4eERGRsvBVMi7YTd26tXuIiIhUd764ZhykCahFRCTy+KNlTPF+6ldegZ07XXd1ixYeBCUiIhIifyTjErqp//xn+OYb6NJFyVhERKo3XyTjPAU7qUeNgpUrXTIWERGpznyVjAu66SavIxAREQmNLwZwBbupNYBLREQijy+ScZ68VLxnDyxZAhs3ehqOiIhISHyRjIuOpl6yBE49FUaO9CggERGRMvBHMi4ymjomBtLSoGdP72ISEREJla8GcOUl47POcg8REZFI4I+WsdcBiIiIVIA/knFuP7XJHcIVCHgZjYiISNn4Ihnny+2mHjAAmjSBzz7zNBoREZGQ+OKacdFu6mXLICMD4uI8CUdERKRM/JGM80ZT577ftw8WLYKuXT0LSUREJGS+SMZ58mbgiomBgQM9DkZERCREPrlmrPHUIiISuXyRjAt2U//4IwwfDnfc4WlIIiIiIfNZNzWsWwdvvgl793odjYiISGhCahkbY4YaY1YbY9YYY249Rr1TjF/Va7oAABunSURBVDE5xphLwhdi6Qp2Uvfs6ZLxn/5UlRGIiIiUX6ktY2NMNPAEcDawCVhqjHnLWruyhHoPAAsrI9BjCXZTG447DoYNq+oIREREyi+UlnEvYI21dq21NhOYA5SU7n4PvA5sC2N8ZaLljEVEJBKFkoxTgIIrA2/KLctnjEkBhgMzj7UhY8xoY8wyY8yy7du3lzXWo8qbDhPgvfdg1ixYuzZsmxcREalUoSTjktqbRe8lehiYbK3NOdaGrLVPWWvTrLVpjRs3DjXGkBngmWfg+uvhv/8N++ZFREQqRSijqTcBLQu8bwGkF6mTBszJnXSjEXCuMSbbWvtmWKIsgzPPhKQkOOGEqt6ziIhI+YSSjJcC7Y0xbYDNwG+AywtWsNa2yXttjJkNvONFIgYYO9Y9REREIkWpydham22MGY8bJR0NPGut/c4YMzb382NeJxYREZFjC2nSD2vtfGB+kbISk7C19tqKh1U+OTmwciXUqQMtW5ZeX0REpDrwxXSYefbshS5dIC3N60hERERC56vpMHNyoFMnaNjQ60hERERC56tkfFxjWLXK6yhERETKxlfd1CIiIpHId8nYamljERGJML5Ixnn597XX3YQfU6Z4Go6IiEiZ+CIZF5SRAYsXex2FiIhI6Hw1gOu8c+H3l7nWsYiISKTwVTJOrAVt2pReT0REpDrxXTe1iIhIpPFVMn7lZbd84pIlXkciIiISOl8l4y+/hFmzYNkyryMREREJna+uGV96KfRpC/37ex2JiIhI6HyVjE86GcZf5nUUIiIiZeOrbmoREZFI5ItknDcF5jdfw4IFsH+/t/GIiIiUhS+ScZ6HHzGccw78/LPXkYiIiITOV9eMu3WF1HhITvY6EhERkdD5KhlPmgSnHu91FCIiImXjq25qERGRSOSrZJydrfWMRUQk8vgqGZ99NsTFKSGLiEhk8dU14+hoiI4DY7yOREREJHS+ahm//z7s3et1FCIiImXjq2QMEOOrtr6IiNQEvkvGIiIikcYXydjiRmzdfDPcfrvHwYiIiJSRrzp1ly2DpINeRyEiIlI2vkrGf/0r9G7jdRQiIiJl46tk3KsX9G7rdRQiIiJl44trxiIiIpHMV8n4jTfg44+9jkJERKRsfJWMH30UXnnF6yhERETKxlfXjIcPh/6nex2FiIhI2fgqGd90E/TRAC4REYkwvuqmFhERiUS+SsYbNsDu3V5HISIiUjb+SMa56xdffTU8/7y3oYiIiJSVP5JxrtSW0LCh11GIiIiUja8GcP3f/2kGLhERiTy+ahmLiIhEIiVjERERj/kqGY8YAf/+t9dRiIiIlI2vknF6Ohw+7HUUIiIiZeOrZPzXv0JamtdRiIiIlI2vknGv3tC8uddRiIiIlI2vkrHxOgAREZFy8FUyfvhh2LzZ6yhERETKxlfJeO5c2LnT6yhERETKxlfJeNIkSE31OgoREZGyCSkZG2OGGmNWG2PWGGNuLeHzK4wx3+Q+PjfG9Ah/qKUbNgySk73Ys4iISPmVmoyNMdHAE8A5QGdghDGmc5Fq64D+1truwD3AU+EOVERExK9CaRn3AtZYa9daazOBOcCwghWstZ9ba/NWEl4CtAhvmKFZvFiTfoiISOQJJRmnABsLvN+UW3Y01wHvViSo8rr1Vti9u/R6IiIi1UkoSyiWdPuuLbGiMWfiknHfo3w+GhgNkFoJI6369IHExLBvVkREpFKF0jLeBLQs8L4FkF60kjGmO/AMMMxaW+INRtbap6y1adbatMaNG5cn3mN64AEN4BIRkcgTSjJeCrQ3xrQxxsQBvwHeKljBGJMKvAFcZa39IfxhioiI+Fep3dTW2mxjzHhgIRANPGut/c4YMzb385nAnUBDYIYxBiDbWlvlSzaU2HcuIiJSzYVyzRhr7XxgfpGymQVeXw9cH97Qym5Af9j3E8TFeR2JiIhI6Hw1AxdAdLTXEYiIiJSNr5LxRx9DlK9+kYiI1AS+Sl0GMFpHUUREIoyvkrGIiEgk8kUyPnzEPU+Z6mkYIiIi5eKLZHwkNxkvX+5tHCIiIuXhi2ScNwXmHXd4G4eIiEh5+CIZx8W6516neBuHiIhIefgiGYuIiEQyXyTjXbnLJn7wgbdxiIiIlEdI02FWd3lrGN99N9z2W29jEZGy27dvH9u2bSMrK8vrUETKLCYmhoSEBBo3bkxCQkL5thHmmDxRt657vt7z2bFFpKz27dvH1q1bSUlJITExEaOZeySCWGvJzs7mwIEDbNiwgSZNmlCvXr0yb8cXybhxI/d81VXexiEiZbdt2zZSUlKoVauW16GIlJkxhtjYWOrXr098fDxbtmwpVzL2xTVjEYlcWVlZJObdnygSwRITEzmSN/FFGfkiGWdmuufde7yNQ0TKR13T4gcV+Xfsi2T844/u+U+3exuHiIhIefgiGUfnXvmuX9/bOERERMrDF8m4U0f3fO+93sYhIgLw5ptv0q9fP4477jgSExNp1aoVF154IQsWLChWd+XKlYwaNYo2bdqQkJBA7dq16dGjB5MmTWLNmjWF6rZu3RpjDMYYYmJiaNiwIb179+bWW2/l559/rqJfJ5XBF8lYRKS6ePTRRxk+fDjt27dn1qxZzJs3jz/96U8AfPjhh4XqzpkzhxNPPJGvv/6aW2+9lQULFvDGG2/w61//mjfffJPzzz+/2PaHDBnC4sWL+eSTT/jnP//JBRdcwJw5c+jatStz586tkt8olcBa68nj5JNPtuHy2Y/bbavJ79glP+0I2zZFpGqsXLnS6xDCqmXLlvbCCy8s8bOcnJz816tWrbLx8fH24osvtllZWcXqZmZm2pkzZxYqa9Wqlb3iiiuK1d2/f7897bTTbGJiot24cWMFf4FURGn/noFltoSc6IuW8bcr3PNDD3sbh4jIrl27aNq0aYmfRUUFT7kPP/wwgUCAJ554gpiY4lM+xMbGMmbMmJD2Wbt2bWbMmEFGRgZPPvlk+QIXT/li0o/9+93ztq3exiEi0qtXL55//nnatm3LsGHD6NChQ4n1PvjgA0455RSaNGkSlv326NGD5s2b89lnn4Vle1K1fNEy7tHDPf/+Rm/jEJHwMcY9Cjr/fFf29tvBsqeecmWjRwfL0tNdWfPmhb9/8smu/IsvgmVTp7qyqVODZQU/L6uZM2fSrl07brnlFjp27EijRo0YMWIE//73vwvV27RpE6mpqcW+n5OTQ3Z2dv6jLFJTU/nll1/KH7x4xhfJOCl3Fr3mzbyNQ0SkQ4cOfPnll3z88cfcfvvt9OzZk7lz5zJkyBDuDeGWj6SkJGJjY/MfRUdUH4u1VhOoRChfdFOLiP9YW7ysYIs4z+jRhVvF4FrEJX2/pBbv1KmFW8XgWtAVER0dTb9+/ejXrx8A6enpDB06lLvuuotx48ZRv359WrRowYYNG4p99/PPPycQCPDOO+9w1113lWm/GzdupFOnThULXjzhi5bx2nXu+YMPj11PRMQLzZs35/rrryc7O5sfc6cMHDhwIEuXLmXbtm2F6p500kmkpaXRunXrMu3jq6++Ij09nb59+4YrbKlCvkjG69e7508+8TYOEZGNGzeWWP79998D5I+0njBhAsYYxo0bR05OToX2eeDAAcaNG0etWrVCHoEt1YsvuqnbtAHWwJlneh2JiNR0Xbt25cwzz2T48OG0adOGffv2MX/+fGbOnMlll12WP2jrhBNO4Nlnn2XkyJH07t2b3/72t3Ts2JGcnBzWrVvHU089RWxsLPHx8YW2v2PHDpYsWYK1lr1797J8+XKefvpptm/fzksvvUTzoqPWJCL4Ihm3buWe+/fzNg4RkQceeID58+dz5513snXrVqKjo+nQoQN/+ctfmDBhQqG6V1xxBd27d2f69Oncd999bNmyhdjYWI4//njOOussXnzxRVq2bFnoOwsXLmThwoVERUVRt25d2rVrx69//WtuuOEGWrVqVZU/VcLIF8lYRKS6GDt2LGPHjg25frdu3XjuuedCqqv5p/3LF9eM89YxTk/3Ng4REZHy8EUy/u8S9/yPf3gbh4iISHn4IhnXreeem6d4G4eIiEh5+CIZn3aqe776Km/jEBERKQ9fJGMREZFIpmQsIiLiMV8k43nz3fPMmd7GISIiUh6+SMaZme758GFv4xARESkPXyTj8851z5qSVUREIpEvknFcnHtOTPQ2DhERkfLwRTIWEakuZs+ejTEm/xEXF8fxxx/PbbfdxmGPrqVNnToVY4wn+y6q6PEp+Hj//fe9Dq+Y2bNn8+yzz1b6fnwxN/X/lrrnDz6A3m29jUVEBODVV1+lRYsW7N+/n7lz53L//fezf/9+HnvsMa9Dqxbyjk9BnTt39iiao5s9ezbZ2dmMGjWqUvfji2ScN3f696s9DUNEJF/Pnj1p164dAGeffTY//vgjs2bN4pFHHiEqSp2SBY9POB05cqTYspORwBf/Ik45xT0PGuhtHCIiR3PSSSeRkZHBjh078su2b9/OmDFj6NChA7Vq1aJly5ZcfvnlbN68udB387qZf/zxR8477zxq165Nq1atuPvuuwkEAoXqfvnll5xxxhkkJCSQkpLCPffcg7W2WDz79u1j/PjxNG/enPj4eDp27MhDDz1UqO5HH32EMYY333yTMWPG0KBBA+rXr8/EiRPJyclh6dKl9O3bl6SkJLp06cLChQvDdrxWr17N8OHDSU5OJjExkT59+rBgwYISj8uKFSsYMmQItWvX5rLLLsv//I033qBPnz7UqlWL5ORkLr30UjZs2FBoGy+++CInnngitWvXpl69enTr1o0nn3wSgAEDBvDxxx/z2Wef5XelDxgwIGy/sSBftIzbtHbPnTp5GYWIyNH9/PPP1KtXj4YNG+aX7dq1i4SEBO6//34aN25Meno606ZN4/TTT+f7778nISGh0DaGDx/OyJEjmThxIm+//TZTpkyhZcuWjBw5EoAdO3YwcOBAmjZtyvPPP098fDwPPvhgsQQUCAQ477zzWL58OXfffTfdunVj3rx5TJo0ie3bt3PfffcVqj9hwgQuuugiXn75ZRYtWsS9995LdnY277//PjfffDMpKSnce++9XHTRRaxfv55GjRqVejxycnLIzs7Of2+MITo6GoD09HT69u1LnTp1ePzxx6lXrx5PPPEE5513Hu+88w7nnHNOoW0NGzaM6667jsmTJ+f3OsycOZMbbriBkSNHcuedd7J//36mTp1K//79+eabb6hTpw6ffvopV155JTfeeCMPPvgggUCA77//nj173FKAM2bM4MorryQnJyc/QdetW7fU31YevkjGIuIvd739HSvT93kaQ+fmdZlyfpdyfz8v2eRdM3799dd5+OGH8xMOQMeOHXnkkUcKfef0008nNTWVd999l+HDhxfa5h/+8If8xHvWWWfx4Ycf8tJLL+WXPfTQQxw8eJCFCxeSmpoKuC7yVq1aFdrO/Pnz+fTTT3nuuee49tprARg8eDAHDx5k2rRpTJo0qVBCHThwINOnT8/f3rx583j88cf55JNP6Nu3LwDNmjWjR48ezJs3j2uuuabU49OpSOvp9NNP59NPPwVg+vTp7N69m8WLF+d3ZZ977rl07tyZ22+/vVgyvvHGG7npppvy3x84cIDJkyczcuTIQoOvevfuTYcOHZg1axYTJkxgyZIlJCcn8/DDD+fXGTx4cP7rzp07U7duXbKzs+nTp0+pv6kifNFNvT73j75167yNQ0QkT6dOnYiNjaVBgwZcd911jBkzhvHjxxer9/e//50ePXpQu3ZtYmJi8pPo6tXFB8Gcd955hd537dq1UKt38eLF9OnTJ38bAElJSZx//vmFvrdo0SKioqIYMWJEofIrr7ySzMxMFi9eXKi8aPLr1KkTSUlJ+Yk4rwxg48aNxQ9GCebOncvSpUvzH7NmzSoUX58+fQpdU46OjmbEiBF89dVX7NtX+A+1on+0LF68mH379nHFFVeQnZ2d/2jRogWdOnVi0aJFAJxyyins3r2bK6+8knfeeSe/RewFX7SMc/+Y4r334DeDvI1FRCquIi3S6mLu3Lm0aNGC7du3M336dGbMmEHv3r25+uqr8+s89thj3HjjjUyaNIkHH3yQ+vXrEwgE6NOnT4m3QTVo0KDQ+/j4+EL1fvnlF7p27Vrse02aNCn0fteuXTRo0KDYQKemTZvmf15Q/fr1C72Pi4sjOTm5WBkQ8u1bXbt2PeoArl27dnHiiScWK2/atCnWWnbv3l2ou7hZs2aF6m3btg1wvQclyfs9/fv359VXX+Wxxx7LT+j9+/dn+vTpdO/ePaTfES6+SMapqcD30Ea3NYlINVEw2QwcOJDu3btz8803c/HFF5OUlATAnDlzGDRoENOmTcv/3roKdPE1a9aMrVu3FisvWtagQQN27dpFZmZmfhIF2LJlC0Ch69peaNCgQX4sBW3ZsgVjTLE/SoreQ50X/+zZs+nSpfgfdnXq1Ml/fckll3DJJZdw4MABPvroIyZPnszQoUPZtGlTlY5690U39Rm5PSVnl/xHkIiIp/IGUm3bto0ZM2bklx86dIjY2NhCdZ977rly7+fUU09lyZIlhbqKDx48yNtvv12oXv/+/QkEArz66quFyl944QXi4uIq/fpoafr378+SJUv4Oe++Vdz19JdffpkTTzyxUDItyWmnnUadOnVYs2YNaWlpxR4dO3Ys9p3atWvzq1/9ijFjxvDLL7+wc+dOwP23y8jICOvvK4kvWsYiItXdBRdcwCmnnMLf/vY3xo8fT2JiIkOHDuWBBx7gvvvuo1evXnz44Ye89tpr5d7HxIkTmTFjBoMHD2bq1Kn5fwQkFpkr+JxzzqFv376MHTuW7du306VLF+bPn88zzzzDH//4x5BGQ1emiRMnMnv2bM4++2zuuusu6taty4wZM/jhhx+YN29eqd+vW7cuDz74IOPGjWP79u2cc8451KtXj82bN/Pxxx8zYMAALr/8cu688062bt3KmWeeSfPmzdm0aROPPvooPXv2pHHjxoAbxDVjxgxefvlljj/+eOrUqVNiMq+okJKxMWYo8AgQDTxjrf1Lkc9N7ufnAoeAa621y8Mc61Fl57jnIrfbiYhUK/feey9Dhgxh5syZTJw4kTvvvJM9e/bw0EMPcfjwYfr378/ChQtp27Z819waNWrEBx98wE033cQ111xDw4YNGTt2LNnZ2dx999359aKiopg3bx633XYbDzzwADt37qR169ZMnz6dCRMmhOvnllvz5s359NNPmTx5MjfccANHjhyhZ8+ezJs3j6FDh4a0jTFjxtCyZUsefPBBXnzxRbKyskhJSaFfv3707NkTcKOrH330USZOnMiuXbs47rjjGDx4MPfcc0/+diZPnszq1au5/vrrOXDgAP379+ejjz4K+282Jd0MXqiCMdHAD8DZwCZgKTDCWruyQJ1zgd/jknFv4BFrbe9jbTctLc0uW7asYtHn+u2fdvBe9n85J64Pf7/b22sdIlI2q1at4oQTTvA6DJGwKO3fszHmC2ttWtHyUK4Z9wLWWGvXWmszgTnAsCJ1hgH/sM4SINkY06zohipL3m17UdHHriciIlIdhZKMU4CCN45tyi0rax2MMaONMcuMMcu2b99e1liP6vdjYjmjfSN+Pya29MoiIiLVTCjJuKR1t4r2bYdSB2vtU9baNGttWt7F8XDo0rwe/3ddbzo1rZxpykRERCpTKMl4E9CywPsWQHo56oiIiEgJQknGS4H2xpg2xpg44DfAW0XqvAVcbZw+wF5r7S9hjlVERMSXSr21yVqbbYwZDyzE3dr0rLX2O2PM2NzPZwLzcSOp1+BubRpZeSGLiN9Ya4vNoiQSaUq7O+lYQrrP2Fo7H5dwC5bNLPDaAuPKHYWI1FixsbFkZGRQq1Ytr0MRqZCMjIxi832HyhfTYYpI5DruuOPYvHkzhw4dqlDLQsQL1lqysrLYtWsXmzZtKve83poOU0Q8lbf6Tnp6OllZWR5HI1J2MTExJCQkkJqaSkJCQvm2EeaYRETKrG7duoWWxBOpadRNLSIi4jElYxEREY8pGYuIiHhMyVhERMRjSsYiIiIeUzIWERHxmPHqJntjzHZgfRg32QjYEcbt1VQ6jhWnY1hxOoYVp2NYcZVxDFtZa4stW+hZMg43Y8wya22a13FEOh3HitMxrDgdw4rTMay4qjyG6qYWERHxmJKxiIiIx/yUjJ/yOgCf0HGsOB3DitMxrDgdw4qrsmPom2vGIiIikcpPLWMREZGIFHHJ2Bgz1Biz2hizxhhzawmfG2PMo7mff2OMOcmLOKuzEI7hFbnH7htjzOfGmB5exFmdlXYMC9Q7xRiTY4y5pCrjixShHEdjzABjzFfGmO+MMR9XdYzVXQj/P9czxrxtjPk69xiO9CLO6soY86wxZpsxZsVRPq+anGKtjZgHEA38BLQF4oCvgc5F6pwLvAsYoA/wX6/jrk6PEI/haUD93Nfn6BiW/RgWqPchMB+4xOu4q9sjxH+LycBKIDX3/XFex12dHiEew9uAB3JfNwZ2AXFex15dHkA/4CRgxVE+r5KcEmkt417AGmvtWmttJjAHGFakzjDgH9ZZAiQbY5pVdaDVWKnH0Fr7ubV2d+7bJUCLKo6xugvl3yHA74HXgW1VGVwECeU4Xg68Ya3dAGCt1bEsLJRjaIE6xhgD1MYl4+yqDbP6stYuwh2To6mSnBJpyTgF2Fjg/abcsrLWqcnKenyuw/1VKEGlHkNjTAowHJhZhXFFmlD+LXYA6htjPjLGfGGMubrKoosMoRzDx4ETgHTgW+Ama22gasLzhSrJKTHh3mAlMyWUFR0OHkqdmizk42OMOROXjPtWakSRJ5Rj+DAw2Vqb4xokUoJQjmMMcDIwCEgEFhtjllhrf6js4CJEKMdwCPAVMBA4HnjPGPOJtXZfZQfnE1WSUyItGW8CWhZ43wL3115Z69RkIR0fY0x34BngHGvtziqKLVKEcgzTgDm5ibgRcK4xJtta+2bVhBgRQv3/eYe19iBw0BizCOgBKBk7oRzDkcBfrLsAusYYsw7oBPyvakKMeFWSUyKtm3op0N4Y08YYEwf8BnirSJ23gKtzR8D1AfZaa3+p6kCrsVKPoTEmFXgDuEotkBKVegyttW2sta2tta2B14DfKREXE8r/z/8CzjDGxBhjagG9gVVVHGd1Fsox3IDrWcAY0wToCKyt0igjW5XklIhqGVtrs40x44GFuFGEz1prvzPGjM39fCZu5Oq5wBrgEO6vQskV4jG8E2gIzMht2WVbTTifL8RjKKUI5Thaa1cZYxYA3wAB4BlrbYm3oNREIf5bvAeYbYz5FtflOtlaq9WcchljXgIGAI2MMZuAKUAsVG1O0QxcIiIiHou0bmoRERHfUTIWERHxmJKxiIiIx5SMRUREPKZkLCIi4jElYxEREY8pGYuIiHhMyVhERMRj/x/K9mZaT3NYjQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize = (8,6))\n",
    "plt.plot(fpr, tpr, 'b:', linewidth=2, label ='SGD')\n",
    "plt.plot(fpr_forest, tpr_forest, label ='Random Forest')\n",
    "plt.legend(loc='lower right', fontsize=16)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9885960849419336"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rand 比 SGD效果好，ROC AUC 分数高很多\n",
    "roc_auc_score(y_train_8, y_socres_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9773835920177384"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 精度和召回也很高\n",
    "y_train_pred_forest = cross_val_predict(forest_clf, X_train, y_train_8, cv=3)\n",
    "precision_score(y_train_8, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7533754913689967"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(y_train_8, y_train_pred_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 总结\n",
    "   1. 选择合适的指标利用交叉验证类对分类器进行评估\n",
    "   2. 使用满足需求的精度/召回率权衡\n",
    "   3. 使用ROC曲线和ROC AUC比较多个模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多类别分类器\n",
    "   1. 尝试8 之外的检查\n",
    "   2. 多类别 区分2个以上类别\n",
    "   3. 随机森林和朴素贝叶斯可以直接处理多个类别\n",
    "   4. 支持向量机svm和线性分类器只可以处理二元分类器\n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 解决方案\n",
    "   1. 将图片分类0-9， 训练10个二元分类器，每个数字一个，检查一张图片时，获取每个分类器的决策分数，称为一对多 OvA\n",
    "   2. 将每对数字训练一个二选分类器，区分0,1  区分0,2 区分1,2 ，称为1对1 OvO策略，存在N个类别，需要N*（N-1）/2 个分类器，最后看那个分类器获胜最多。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 优缺点\n",
    "   1. OvO只需要用到部分训练集对其必须区分两个类别进行训练\n",
    "   2. 对于较小训练集集合OvO比较有优势，大训练集合OvA速度快，所以OvA更常用，比如SVM在数据规模扩大时表现糟糕\n",
    "   3. sklearn检查到使用二元分类算法进行多类别分类任务，会自动运行OvA，SVM分类器除外"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['8'], dtype='<U1')"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.fit(X_train, y_train)\n",
    "sgd_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-38094.21968389, -14201.58901526,  -8886.67389403,\n",
       "         -9794.61559481, -19843.29977604,  -6095.10616217,\n",
       "        -36516.93635076, -22231.2699121 ,   5119.62495739,\n",
       "         -1261.58996052]])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 内部实际上训练了10个二元分类器，获得图片的决策分数，然后选择分数最高的类别\n",
    "# 返回10个分数，每个类别1个\n",
    "some_digit_socres = sgd_clf.decision_function([some_digit])\n",
    "some_digit_socres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argmax(some_digit_socres)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'], dtype='<U1')"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 目标类别列表会存储在classes_这个属性中，按照值大小排序\n",
    "sgd_clf.classes_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'8'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_clf.classes_[np.argmax(some_digit_socres)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['8'], dtype=object)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用OVO策略，一对一或者一对多\n",
    "from sklearn.multiclass import OneVsOneClassifier\n",
    "ovo_clf = OneVsOneClassifier(SGDClassifier(max_iter=5, tol=-np.infty, random_state=42))\n",
    "ovo_clf.fit(X_train, y_train)\n",
    "ovo_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ovo_clf.estimators_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['8'], dtype=object)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机森林\n",
    "forest_clf.fit(X_train, y_train)\n",
    "forest_clf.predict([some_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0., 0., 0., 0., 0., 1., 0.]])"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机是哪里直接将实例分为多个类别，调用predict_proba()获取分类器将每个实例分类为每个类别的概率\n",
    "forest_clf.predict_proba([some_digit])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 评估分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.88865, 0.873  , 0.8677 ])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用交叉验证的方式评估sgd的准确率\n",
    "cross_val_score(sgd_clf, X_train, y_train, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:557: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:557: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:557: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.9021 , 0.90675, 0.9036 ])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 将输入进行简单缩放，可以得到准确率 90% 以上\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "X_train_scaler = scaler.fit_transform(X_train.astype(np.float64))\n",
    "cross_val_score(sgd_clf, X_train_scaler, y_train, cv=3, scoring='accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 错误分析"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 项目流程\n",
    "   1. 探索数据准备的选项\n",
    "   2. 尝试多个模型\n",
    "   3. 选择最佳模型并用GridSearchCV对参数进行微调\n",
    "   4.尽可能自动化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 确认合适的模型，进一步优化\n",
    "   1. 查看混淆矩阵\n",
    "   2. 使用cross_val_predict(）进行预测\n",
    "   3. 调用confusion_matrix()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:557: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:557: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n",
      "c:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\linear_model\\_stochastic_gradient.py:557: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Mix of label input types (string and number)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-88-44c4512e3f4d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0my_trian_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcross_val_predict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msgd_clf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_train_scaler\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mconf_mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mconf_mx\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\metrics\\_classification.py\u001b[0m in \u001b[0;36mconfusion_matrix\u001b[1;34m(y_true, y_pred, labels, sample_weight, normalize)\u001b[0m\n\u001b[0;32m    271\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    272\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlabels\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 273\u001b[1;33m         \u001b[0mlabels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0munique_labels\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    274\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    275\u001b[0m         \u001b[0mlabels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\李少斌\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\sklearn\\utils\\multiclass.py\u001b[0m in \u001b[0;36munique_labels\u001b[1;34m(*ys)\u001b[0m\n\u001b[0;32m     98\u001b[0m     \u001b[1;31m# Check that we don't mix string type with number type\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     99\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mlabel\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mys_labels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 100\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Mix of label input types (string and number)\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    102\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mys_labels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Mix of label input types (string and number)"
     ]
    }
   ],
   "source": [
    "y_trian_pred = cross_val_predict(sgd_clf, X_train_scaler, y_train, cv=3)\n",
    "conf_mx = confusion_matrix(y_train, y_train_pred)\n",
    "conf_mx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用matplotlib 查看混淆矩阵的图像\n",
    "plt.matshow(conf_mx, cmp = plt.cm.gray)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 大多数在主对角线上，说明他被正确分类\n",
    "# 暗色说明图片较少，分类器在数字上执行效果不好\n",
    "# 假设将焦点放在错误上，为取得错误率，而不是错误的绝对值，需要将混淆矩阵中每个值除以相应类别中的图片数量\n",
    "\n",
    "row_sums = conf_mx.sum(axis = 1 , keepdims =True)\n",
    "norm_conf_mx = conf_mx/ row_sums"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用0填充对角线，只保留错误，重新绘制\n",
    "np.fill_diagonal(norm_conf_mx, 0)\n",
    "plt.mashow(norm_conf_mx, cmap = plt.cm.gray)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 每行代表实际类别，每列代表预测类型\n",
    "# 比较亮，说明许多图片被错误的分类为数字XX\n",
    "# 类别XX 行也比较亮，说明数字XX经常与其他数字混淆\n",
    "# 数字XX 是错误最多的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 结论\n",
    "   1. 改进数字X和X的分类\n",
    "   2. 修正3和5的混淆"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 如何优化分类器\n",
    "   1. 尝试多收集这些数字的训练集\n",
    "   2. 开放新的特征来改进分类器\n",
    "   3. 优化分类器的算法\n",
    "   4. 使用pllow护照opencv对图片预处理，让显示模型更突出\n",
    "   5. 分析单个错误\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 多标签分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 为每个实例产生多个类别，例如照片识别多个人脸\n",
    "# 分类器经过训练可以识别小红，小白，小军，一张照片里有小红，小白\n",
    "# 经过分类器，应该输出（1,1,0），是小红，小白，不是小军\n",
    "# 输出多个二元标签的分类系统称为多标签分类系统"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "y_train_large = (y_train >= 7)  # 实际标签是789\n",
    "y_train_odd = (t_train %2 = =1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "\n",
    "knn_clf = KNeighborsClassifier()\n",
    "knn_clf.fit(X_train, y_multilabel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# knn支持多标签分类，并不是所有的分类器都支持多标签分类\n",
    "knn_clf.predict([sonme_digit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 评估多标签分类器的方法很多，方法之一就是测量每个标签的F1分数，或者其他二元分类器的指标，然后简单平均\n",
    "y_train_knn_pred = cross_val_predict(knn_clf, X_train, y_multilabel, cv=3)\n",
    "f1_score(y_multilabel, y_train_knn_pred, average='macro')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 上面假设了所有标签同等重要，也可以给每个标签设置权重，设置average='weighted'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 多分类输出\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 例子：构建一个系统去除图片中的噪声，输入一张有噪声的图片，它将输入一张干净的数字图片，分类器输出是多个标签，一个像素一个标签，每个标签多个值0到255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 增加噪声，目标奖图片还原为原始图片，创建训练集和测试集\n",
    "\n",
    "noise = np.random.randint(0, 100, (len(X_train), 784))\n",
    "X_train_mod = X_trian + nnoise\n",
    "noise = np.random.randint(0, 100, (len(X_test),784))\n",
    "X_test_mod = X_tset + noise\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_tset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "some_index = 5500\n",
    "plt.subplot(121);plt.imshow(X_test_mod[some_index].reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.subplot(122);plt.inshow(y_test_mod[some_index].reshape(28, 28), cmap = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_clf,fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn.clf.predict([X_test_mod[some_index]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.inshow(clean_digit.reshape(28,28), cmp = matplotlib.cm.binary)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
